[ { "path": ".flake8", "content": "[flake8]\nignore = C408, E121, E123, E126, E226, E24, E704, W503, W504, W605\nexclude = docs/source/conf.py\ninline-quotes = \"\nno-avoid-escape = 1\n" }, { "path": ".github/ISSUE_TEMPLATE/bug-performance-issue--custom-images-.md", "content": "---\nname: Bug/Performance Issue [Custom Images]\nabout: You are testing your own images in software.\ntitle: 'Issue: Bug/Performance Issue [Custom Images]'\nlabels: ''\nassignees: ''\n\n---\n\n**System information**\n- OS Platform and Distribution (e.g., Linux Ubuntu 16.04):\n- Python version:\n- Installed using pip or ROS:\n- Camera:\n- GPU model (if applicable):\n\n**Describe what you are trying to do**\n\n**Describe current behavior**\n\n**Describe the expected behavior** \n\n**Describe the input images**\nProvide details such as original captured resolution, pre-processing, final resolution, etc.\n\n**Describe the physical camera setup**\nProvide details such as the distance from camera to workspace, orientation, etc.. Attach a picture if possible.\n\n**Other info / logs**\nInclude any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached.\n" }, { "path": ".github/ISSUE_TEMPLATE/bug-performance-issue--physical-robot-.md", "content": "---\nname: Bug/Performance Issue [Physical Robot]\nabout: You are running on a physical robot.\ntitle: 'Issue: Bug/Performance Issue [Physical Robot]'\nlabels: ''\nassignees: ''\n\n---\n\n**System information**\n- OS Platform and Distribution (e.g., Linux Ubuntu 16.04):\n- Python version:\n- Installed using pip or ROS:\n- Camera:\n- Gripper: \n- Robot:\n- GPU model (if applicable):\n\n**Describe what you are trying to do**\nProvide details such as what angle you are attempting grasps at relative to the workspace.\n\n**Describe current behavior**\n\n**Describe the expected behavior** \n\n**Describe the input images**\nProvide details such as original captured resolution, pre-processing, final resolution, etc.\n\n**Describe the physical camera setup**\nProvide details such as the distance from camera to workspace, orientation, etc.. Attach a picture if possible.\n\n**Describe the physical robot setup**\nAttach a picture if possible.\n\n**Other info / logs**\nInclude any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached.\n" }, { "path": ".github/ISSUE_TEMPLATE/bug-performance-issue--replication-.md", "content": "---\nname: Bug/Performance Issue [Replication]\nabout: You are replicating training or the policy with the provided images/datasets/models.\ntitle: 'Issue: Bug/Performance Issue [Replication]'\nlabels: ''\nassignees: ''\n\n---\n\n**System information**\n- OS Platform and Distribution (e.g., Linux Ubuntu 16.04):\n- Python version:\n- Installed using pip or ROS:\n- GPU model (if applicable):\n\n**Describe the result you are trying to replicate**\nIf you can, provide a link to the section in the [documentation](https://berkeleyautomation.github.io/gqcnn/index.html).\n\n**Provide the exact sequence of commands / steps that you executed to replicate this result**\n\n**Describe the unexpected behavior** \n\n**Other info / logs**\nInclude any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached.\n" }, { "path": ".github/ISSUE_TEMPLATE/installation-issue.md", "content": "---\nname: Installation Issue\nabout: You are experiencing installation issues.\ntitle: 'Issue: Installation Issue'\nlabels: ''\nassignees: ''\n\n---\n\n**System information**\n- OS Platform and Distribution (e.g., Linux Ubuntu 16.04):\n- Python version:\n- Installed using pip or ROS:\n\n**Describe the problem**\n\n**Provide the exact sequence of commands / steps that you executed before running into the problem**\n\n**Any other info / logs**\nInclude any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached.\n" }, { "path": ".github/ISSUE_TEMPLATE/other-issue.md", "content": "---\nname: Other Issue\nabout: Your issue doesn't fall into the above categories.\ntitle: 'Issue: Other Issue'\nlabels: ''\nassignees: ''\n\n---\n\n**System information**\n- OS Platform and Distribution (e.g., Linux Ubuntu 16.04):\n- Python version:\n- Installed using pip or ROS:\n\n**What is the problem**\nProvide as much detail as possible and steps to replicate it if applicable.\n\n**Other info / logs**\nInclude any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached.\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\ndata/training/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\n*.egg-info/\n.installed.cfg\n*.egg\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*,cover\n.hypothesis/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# PyBuilder\ntarget/\n\n# IPython Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# dotenv\n.env\n\n# virtualenv\nvenv/\nENV/\n\n# Spyder project settings\n.spyderproject\n\n# Rope project settings\n.ropeproject\n\n# Temp files\n*~\n.#*\n#*" }, { "path": ".style.yapf", "content": "[style]\nbased_on_style=pep8\nallow_split_before_dict_value=False\njoin_multiple_lines=False\n" }, { "path": ".travis.yml", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\nlanguage: python\n\nmatrix:\n - name: \"Python 3.5 on Xenial Linux\"\n python: 3.5\n dist: xenial\n\n - name: \"Python 3.6 on Xenial Linux\"\n python: 3.6\n dist: xenial\n\n - name: \"Python 3.7 on Xenial Linux\"\n python: 3.7\n dist: xenial\n\n - name: \"Linter\"\n python: 3.7\n dist: xenial\n before_install: []\n install: []\n script:\n - pip install yapf==0.27.0 # Must use specific version.\n - pip install flake8 flake8-comprehensions flake8-quotes==2.0.0\n - ./ci/travis/format.sh --all # Should exit with 0 for no diffs.\n - flake8\n\nbefore_install:\n - sudo apt-get update\n - sudo apt-get install -y curl g++ make\n - sudo apt-get install -y python-opengl # For GLU.\n - pushd ~\n - curl -L http://download.osgeo.org/libspatialindex/spatialindex-src-1.8.5.tar.gz | tar xz\n - cd spatialindex-src-1.8.5\n - ./configure\n - make\n - sudo make install\n - sudo ldconfig\n - popd\n - pip install -U setuptools # Required for easy_install to find right\n # skimage version for Python 3.5.\n\ninstall:\n - pip install .\n\nscript:\n - python -c \"import gqcnn\"\n\nnotifications:\n email: false\n" }, { "path": "CMakeLists.txt", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\ncmake_minimum_required(VERSION 2.8.3)\nproject(gqcnn)\n\n## Find catkin macros and libraries\n## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz)\n## is used, also find other catkin packages\nfind_package(catkin REQUIRED COMPONENTS\n rospy\n message_generation\n geometry_msgs\n sensor_msgs\n)\n\n## System dependencies are found with CMake's conventions\n# find_package(Boost REQUIRED COMPONENTS system)\n\n## Uncomment this if the package has a setup.py. This macro ensures\n## modules and global scripts declared therein get installed\n## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html\ncatkin_python_setup()\n\n################################################\n## Declare ROS messages, services and actions ##\n################################################\n\n## To declare and build messages, services or actions from within this\n## package, follow these steps:\n## * Let MSG_DEP_SET be the set of packages whose message types you use in\n## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...).\n## * In the file package.xml:\n## * add a build_depend tag for \"message_generation\"\n## * add a build_depend and a run_depend tag for each package in MSG_DEP_SET\n## * If MSG_DEP_SET isn't empty the following dependency has been pulled in\n## but can be declared for certainty nonetheless:\n## * add a run_depend tag for \"message_runtime\"\n## * In this file (CMakeLists.txt):\n## * add \"message_generation\" and every package in MSG_DEP_SET to\n## find_package(catkin REQUIRED COMPONENTS ...)\n## * add \"message_runtime\" and every package in MSG_DEP_SET to\n## catkin_package(CATKIN_DEPENDS ...)\n## * uncomment the add_*_files sections below as needed\n## and list every .msg/.srv/.action file to be processed\n## * uncomment the generate_messages entry below\n## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...)\n\n## Generate messages in the 'msg' folder\n add_message_files(\n FILES\n GQCNNGrasp.msg\n BoundingBox.msg\n Action.msg\n Observation.msg\n )\n\n## Generate services in the 'srv' folder\n add_service_files(\n FILES\n GQCNNGraspPlanner.srv\n GQCNNGraspPlannerBoundingBox.srv\n GQCNNGraspPlannerSegmask.srv\n )\n\n## Generate actions in the 'action' folder\n# add_action_files(\n# FILES\n# Action1.action\n# Action2.action\n# )\n\n## Generate added messages and services with any dependencies listed here\n generate_messages(\n DEPENDENCIES\n geometry_msgs\n sensor_msgs\n)\n\n################################################\n## Declare ROS dynamic reconfigure parameters ##\n################################################\n\n## To declare and build dynamic reconfigure parameters within this\n## package, follow these steps:\n## * In the file package.xml:\n## * add a build_depend and a run_depend tag for \"dynamic_reconfigure\"\n## * In this file (CMakeLists.txt):\n## * add \"dynamic_reconfigure\" to\n## find_package(catkin REQUIRED COMPONENTS ...)\n## * uncomment the \"generate_dynamic_reconfigure_options\" section below\n## and list every .cfg file to be processed\n\n## Generate dynamic reconfigure parameters in the 'cfg' folder\n# generate_dynamic_reconfigure_options(\n# cfg/DynReconf1.cfg\n# cfg/DynReconf2.cfg\n# )\n\n###################################\n## catkin specific configuration ##\n###################################\n## The catkin_package macro generates cmake config files for your package\n## Declare things to be passed to dependent projects\n## INCLUDE_DIRS: uncomment this if you package contains header files\n## LIBRARIES: libraries you create in this project that dependent projects also need\n## CATKIN_DEPENDS: catkin_packages dependent projects also need\n## DEPENDS: system dependencies of this project that dependent projects also need\ncatkin_package(\n# INCLUDE_DIRS include\n# LIBRARIES yumipy\n# CATKIN_DEPENDS rospy\n# DEPENDS system_lib\n CATKIN_DEPENDS message_runtime\n)\n\n###########\n## Build ##\n###########\n\n## Specify additional locations of header files\n## Your package locations should be listed before other locations\n# include_directories(include)\ninclude_directories(\n ${catkin_INCLUDE_DIRS}\n)\n\n## Declare a C++ library\n# add_library(yumipy\n# src/${PROJECT_NAME}/yumipy.cpp\n# )\n\n## Add cmake target dependencies of the library\n## as an example, code may need to be generated before libraries\n## either from message generation or dynamic reconfigure\n# add_dependencies(yumipy ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})\n\n## Declare a C++ executable\n# add_executable(yumipy_node src/yumipy_node.cpp)\n\n## Add cmake target dependencies of the executable\n## same as for the library above\n#add_dependencies(gqcnn gqcnn_generate_messages_cpp)\n\n## Specify libraries to link a library or executable target against\n# target_link_libraries(yumipy_node\n# ${catkin_LIBRARIES}\n# )\n\n#############\n## Install ##\n#############\n\n# all install targets should use catkin DESTINATION variables\n# See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html\n\n## Mark executable scripts (Python etc.) for installation\n## in contrast to setup.py, you can choose the destination\n# install(PROGRAMS\n# scripts/my_python_script\n# DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}\n# )\n\n## Mark executables and/or libraries for installation\n# install(TARGETS yumipy yumipy_node\n# ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}\n# LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}\n# RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}\n# )\n\n## Mark cpp header files for installation\n# install(DIRECTORY include/${PROJECT_NAME}/\n# DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION}\n# FILES_MATCHING PATTERN \"*.h\"\n# PATTERN \".svn\" EXCLUDE\n# )\n\n## Mark other files for installation (e.g. launch and bag files, etc.)\n# install(FILES\n# # myfile1\n# # myfile2\n# DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}\n# )\n\n#############\n## Testing ##\n#############\n\n## Add gtest based cpp test target and link libraries\n# catkin_add_gtest(${PROJECT_NAME}-test test/test_yumipy.cpp)\n# if(TARGET ${PROJECT_NAME}-test)\n# target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME})\n# endif()\n\n## Add folders to be run by python nosetests\n# catkin_add_nosetests(test)\n" }, { "path": "LICENSE", "content": "Copyright ©2017. The Regents of the University of California (Regents). All Rights Reserved.\nPermission to use, copy, modify, and distribute this software and its documentation for educational,\nresearch, and not-for-profit purposes, without fee and without a signed licensing agreement, is\nhereby granted, provided that the above copyright notice, this paragraph and the following two\nparagraphs appear in all copies, modifications, and distributions. Contact The Office of Technology\nLicensing, UC Berkeley, 2150 Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-\n7201, otl@berkeley.edu, http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n" }, { "path": "README.md", "content": "## Note: Python 2.x support has officially been dropped.\n\n# Berkeley AUTOLAB's GQCNN Package\n

\n \n \"Build\n \n \n \"Release\"\n \n \n \"Software\n \n \n \"Python\n \n

\n\n## Package Overview\nThe gqcnn Python package is for training and analysis of Grasp Quality Convolutional Neural Networks (GQ-CNNs). It is part of the ongoing [Dexterity-Network (Dex-Net)](https://berkeleyautomation.github.io/dex-net/) project created and maintained by the [AUTOLAB](https://autolab.berkeley.edu) at UC Berkeley.\n\n## Installation and Usage\nPlease see the [docs](https://berkeleyautomation.github.io/gqcnn/) for installation and usage instructions.\n\n## Citation\nIf you use any part of this code in a publication, please cite [the appropriate Dex-Net publication](https://berkeleyautomation.github.io/gqcnn/index.html#academic-use).\n\n" }, { "path": "cfg/examples/antipodal_grasp_sampling.yaml", "content": "# policy params\npolicy:\n # general params\n deterministic: 1\n gripper_width: 0.05\n\n # sampling params\n sampling:\n # type\n type: antipodal_depth\n\n # antipodality\n friction_coef: 0.5\n depth_grad_thresh: 0.0025\n depth_grad_gaussian_sigma: 1.0\n downsample_rate: 4\n max_rejection_samples: 4000\n\n # distance\n max_dist_from_center: 1000\n min_dist_from_boundary: 45\n min_grasp_dist: 10.0\n angle_dist_weight: 5.0\n\n # depth sampling\n depth_sampling_mode: uniform\n depth_samples_per_grasp: 1\n depth_sample_win_height: 1\n depth_sample_win_width: 1\n min_depth_offset: 0.015\n max_depth_offset: 0.05\n\n # metrics\n metric:\n type: zero\n\n # visualization\n vis:\n grasp_sampling: 0\n final_grasp: 1\n vmin: 0.4\n vmax: 1.0\n" }, { "path": "cfg/examples/fc_gqcnn_pj.yaml", "content": "policy:\n type: fully_conv_pj\n\n sampling_method: top_k\n num_depth_bins: 16\n gripper_width: 0.05\n gqcnn_stride: 4\n gqcnn_recep_h: 96\n gqcnn_recep_w: 96\n\n # filtering params\n max_grasps_to_filter: 50\n filter_grasps: 0\n\n # metrics\n metric:\n type: fcgqcnn\n gqcnn_model: /path/to/your/FC-GQ-Image-Wise\n gqcnn_backend: tf\n fully_conv_gqcnn_config:\n im_height: 386\n im_width: 516\n\n # visualization\n policy_vis:\n scale: 0.5\n show_axis: 1\n num_samples: 0\n actions_2d: 0\n actions_3d: 0\n affordance_map: 0\n vis:\n final_grasp: 1\n\n vmin: 0.5\n vmax: 0.8\n\n# image pre-processing before input to policy\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/fc_gqcnn_suction.yaml", "content": "policy:\n type: fully_conv_suction\n\n sampling_method: top_k\n gqcnn_stride: 4\n gqcnn_recep_h: 96\n gqcnn_recep_w: 96\n\n # filtering params\n max_grasps_to_filter: 50\n filter_grasps: 0\n\n # metrics\n metric:\n type: fcgqcnn\n gqcnn_model: /path/to/your/FC-GQ-Image-Wise-Suction\n gqcnn_backend: tf\n fully_conv_gqcnn_config:\n im_height: 386\n im_width: 516\n\n # visualization\n policy_vis:\n scale: 0.5\n show_axis: 1\n num_samples: 0\n actions_2d: 0\n actions_3d: 0\n affordance_map: 0\n vis:\n final_grasp: 1\n\n vmin: 0.5\n vmax: 0.8\n\n# image pre-processing before input to policy\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/gqcnn_pj.yaml", "content": "policy:\n # optimization params\n num_seed_samples: 128\n num_gmm_samples: 64\n num_iters: 3\n gmm_refit_p: 0.25\n gmm_component_frac: 0.4\n gmm_reg_covar: 0.01\n\n # general params\n deterministic: 1\n gripper_width: 0.05\n\n # sampling params\n sampling:\n # type\n type: antipodal_depth\n\n # antipodality\n friction_coef: 1.0\n depth_grad_thresh: 0.0025\n depth_grad_gaussian_sigma: 1.0\n downsample_rate: 4\n max_rejection_samples: 4000\n\n # distance\n max_dist_from_center: 160\n min_dist_from_boundary: 45\n min_grasp_dist: 2.5\n angle_dist_weight: 5.0\n\n # depth sampling\n depth_sampling_mode: uniform\n depth_samples_per_grasp: 3\n depth_sample_win_height: 1\n depth_sample_win_width: 1\n min_depth_offset: 0.015\n max_depth_offset: 0.05\n\n # metrics\n metric:\n type: gqcnn\n gqcnn_model: models/GQCNN-4.0-PJ\n\n crop_height: 96\n crop_width: 96\n\n # visualization\n vis:\n grasp_sampling : 0\n tf_images: 0\n grasp_candidates: 0\n elite_grasps: 0\n grasp_ranking: 0\n grasp_plan: 0\n final_grasp: 1\n\n vmin: 0.5\n vmax: 0.8\n\n k: 25\n\n# image proc params\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/gqcnn_suction.yaml", "content": "policy:\n # optimization params\n num_seed_samples: 200\n num_gmm_samples: 50\n num_iters: 3\n gmm_refit_p: 0.25\n gmm_component_frac: 0.4\n gmm_reg_covar: 0.01\n\n # general params\n deterministic: 1\n gripper_width: 0.05\n crop_height: 96\n crop_width: 96\n\n # sampling params\n sampling:\n # type\n type: suction\n\n # params\n max_suction_dir_optical_axis_angle: 30\n delta_theta: 5\n delta_phi: 5\n sigma_depth: 0.0025\n min_suction_dist: 1.0\n angle_dist_weight: 5.0\n depth_gaussian_sigma: 1.0\n num_grasp_samples: 1000\n\n max_dist_from_center: 260\n max_num_samples: 10000\n\n # metric params\n metric:\n type: gqcnn\n gqcnn_model: models/GQCNN-4.0-SUCTION\n\n crop_height: 96\n crop_width: 96\n\n # visualization\n vis:\n grasp_sampling : 0\n tf_images: 0\n plane: 0\n grasp_candidates: 0\n elite_grasps: 0\n grasp_ranking: 0\n grasp_plan: 0\n grasp_affordance_map: 0\n final_grasp: 1\n\n vmin: 0.5\n vmax: 0.8\n\n k: 25\n\n# image proc params\ninpaint_rescale_factor: 0.5\n\n# detection params\ndetection:\n type: point_cloud_box\n\n foreground_mask_tolerance: 60\n min_pt:\n - 0.205\n - -0.3\n - 0.01\n\n max_pt:\n - 0.65\n - 0.3\n - 0.15\n\n selection_policy: min\n focus: 0\n\n min_contour_area: 250.0\n max_contour_area: 1000000.0\n min_box_area: 250.0\n max_box_area: 1000000.0\n box_padding_px: 15\n\n rescale_factor: 1.0\n interpolation: bilinear\n depth_grad_thresh: 10.0\n contour_dist_thresh: 2.5\n\n point_cloud_mask_only: 1\n\n image_width: 640\n image_height: 480\n\n filter_dim: 1\n" }, { "path": "cfg/examples/replication/dex-net_2.0.yaml", "content": "# policy params\npolicy:\n # optimization params\n num_seed_samples: 250\n num_gmm_samples: 50\n num_iters: 3\n gmm_refit_p: 0.25\n gmm_component_frac: 0.4\n gmm_reg_covar: 0.01\n\n # general params\n deterministic: 1\n gripper_width: 0.05\n\n # sampling params\n sampling:\n # type\n type: antipodal_depth\n\n # antipodality\n friction_coef: 0.8\n depth_grad_thresh: 0.0025\n depth_grad_gaussian_sigma: 1.0\n downsample_rate: 2\n max_rejection_samples: 4000\n\n # distance\n max_dist_from_center: 160\n min_dist_from_boundary: 45\n min_grasp_dist: 2.5\n angle_dist_weight: 5.0\n\n # depth sampling\n depth_sampling_mode: uniform\n depth_samples_per_grasp: 1\n depth_sample_win_height: 1\n depth_sample_win_width: 1\n min_depth_offset: 0.005\n max_depth_offset: 0.04\n\n # metrics\n metric:\n type: gqcnn\n gqcnn_model: models/GQCNN-2.0\n\n crop_height: 96\n crop_width: 96\n\n # visualization\n vis:\n grasp_sampling : 0\n tf_images: 0\n grasp_candidates: 0\n elite_grasps: 0\n grasp_ranking: 0\n grasp_plan: 0\n final_grasp: 1\n\n vmin: 0.0\n vmax: 1.0\n\n k: 25\n\n# image proc params\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/replication/dex-net_2.1.yaml", "content": "# policy params\npolicy:\n # optimization params\n num_seed_samples: 250\n num_gmm_samples: 50\n num_iters: 3\n gmm_refit_p: 0.25\n gmm_component_frac: 0.4\n gmm_reg_covar: 0.01\n\n # general params\n deterministic: 1\n gripper_width: 0.05\n\n # sampling params\n sampling:\n # type\n type: antipodal_depth\n\n # antipodality\n friction_coef: 0.8\n depth_grad_thresh: 0.0025\n depth_grad_gaussian_sigma: 1.0\n downsample_rate: 2\n max_rejection_samples: 4000\n\n # distance\n max_dist_from_center: 160\n min_dist_from_boundary: 45\n min_grasp_dist: 2.5\n angle_dist_weight: 5.0\n\n # depth sampling\n depth_sampling_mode: uniform\n depth_samples_per_grasp: 1\n depth_sample_win_height: 1\n depth_sample_win_width: 1\n min_depth_offset: 0.005\n max_depth_offset: 0.04\n\n # metrics\n metric:\n type: gqcnn\n gqcnn_model: models/GQCNN-2.1\n\n crop_height: 96\n crop_width: 96\n\n # visualization\n vis:\n grasp_sampling : 0\n tf_images: 0\n grasp_candidates: 0\n elite_grasps: 0\n grasp_ranking: 0\n grasp_plan: 0\n final_grasp: 1\n\n vmin: 0.0\n vmax: 1.0\n\n k: 25\n\n# image proc params\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/replication/dex-net_3.0.yaml", "content": "# policy params\npolicy:\n # optimization params\n num_seed_samples: 250\n num_gmm_samples: 50\n num_iters: 3\n gmm_refit_p: 0.25\n gmm_component_frac: 0.4\n gmm_reg_covar: 0.01\n\n # general params\n deterministic: 1\n max_approach_angle: 80\n\n # sampling params\n sampling:\n # type\n type: suction\n\n # params\n max_suction_dir_optical_axis_angle: 30\n delta_theta: 1\n delta_phi: 1\n mean_depth: 0.0025\n sigma_depth: 0.000001\n min_suction_dist: 1.0\n angle_dist_weight: 5.0\n depth_gaussian_sigma: 1.0\n\n max_dist_from_center: 10000000000\n max_num_samples: 10000\n\n num_grasp_samples: 500\n\n # metric params\n metric:\n type: gqcnn\n gqcnn_model: models/GQCNN-3.0\n\n crop_height: 96\n crop_width: 96\n\n # visualization\n vis:\n grasp_sampling : 0\n tf_images: 0\n plane: 0\n grasp_candidates: 0\n elite_grasps: 0\n grasp_ranking: 0\n grasp_plan: 0\n final_grasp: 1\n\n vmin: 0.0\n vmax: 1.0\n\n k: 25\n\n# image proc params\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/replication/dex-net_4.0_fc_pj.yaml", "content": "policy:\n type: fully_conv_pj\n\n sampling_method: top_k\n num_depth_bins: 16\n gripper_width: 0.05\n gqcnn_stride: 4\n gqcnn_recep_h: 96\n gqcnn_recep_w: 96\n\n # filtering params\n max_grasps_to_filter: 50\n filter_grasps: 0\n\n # metrics\n metric:\n type: fcgqcnn\n gqcnn_model: models/FCGQCNN-4.0-PJ\n gqcnn_backend: tf\n fully_conv_gqcnn_config:\n im_height: 480\n im_width: 640\n\n # visualization\n policy_vis:\n scale: 0.5\n show_axis: 1\n num_samples: 0\n actions_2d: 0\n actions_3d: 0\n affordance_map: 0\n vis:\n final_grasp: 1\n\n vmin: 0.0\n vmax: 1.0\n\n# image pre-processing before input to policy\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/replication/dex-net_4.0_fc_suction.yaml", "content": "policy:\n type: fully_conv_suction\n\n sampling_method: top_k\n gqcnn_stride: 4\n gqcnn_recep_h: 96\n gqcnn_recep_w: 96\n\n # filtering params\n max_grasps_to_filter: 50\n filter_grasps: 0\n\n # metrics\n metric:\n type: fcgqcnn\n gqcnn_model: models/FCGQCNN-4.0-SUCTION\n gqcnn_backend: tf\n fully_conv_gqcnn_config:\n im_height: 480\n im_width: 640\n\n # visualization\n policy_vis:\n scale: 0.5\n show_axis: 1\n num_samples: 0\n actions_2d: 0\n actions_3d: 0\n affordance_map: 0\n vis:\n final_grasp: 1\n\n vmin: 0.0\n vmax: 1.0\n\n# image pre-processing before input to policy\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/replication/dex-net_4.0_pj.yaml", "content": "policy:\n # optimization params\n num_seed_samples: 128\n num_gmm_samples: 64\n num_iters: 3\n gmm_refit_p: 0.25\n gmm_component_frac: 0.4\n gmm_reg_covar: 0.01\n\n # general params\n deterministic: 1\n gripper_width: 0.05\n\n # sampling params\n sampling:\n # type\n type: antipodal_depth\n\n # antipodality\n friction_coef: 1.0\n depth_grad_thresh: 0.0025\n depth_grad_gaussian_sigma: 1.0\n downsample_rate: 4\n max_rejection_samples: 4000\n\n # distance\n max_dist_from_center: 160\n min_dist_from_boundary: 45\n min_grasp_dist: 2.5\n angle_dist_weight: 5.0\n\n # depth sampling\n depth_sampling_mode: uniform\n depth_samples_per_grasp: 3\n depth_sample_win_height: 1\n depth_sample_win_width: 1\n min_depth_offset: 0.015\n max_depth_offset: 0.05\n\n # metrics\n metric:\n type: gqcnn\n gqcnn_model: models/GQCNN-4.0-PJ\n\n crop_height: 96\n crop_width: 96\n\n # visualization\n vis:\n grasp_sampling : 0\n tf_images: 0\n grasp_candidates: 0\n elite_grasps: 0\n grasp_ranking: 0\n grasp_plan: 0\n final_grasp: 1\n\n vmin: 0.0\n vmax: 1.0\n\n k: 25\n\n# image proc params\ninpaint_rescale_factor: 0.5\n" }, { "path": "cfg/examples/replication/dex-net_4.0_suction.yaml", "content": "policy:\n # optimization params\n num_seed_samples: 200\n num_gmm_samples: 50\n num_iters: 3\n gmm_refit_p: 0.25\n gmm_component_frac: 0.4\n gmm_reg_covar: 0.01\n\n # general params\n deterministic: 1\n gripper_width: 0.05\n crop_height: 96\n crop_width: 96\n\n # sampling params\n sampling:\n # type\n type: suction\n\n # params\n max_suction_dir_optical_axis_angle: 30\n delta_theta: 5\n delta_phi: 5\n sigma_depth: 0.0025\n min_suction_dist: 1.0\n angle_dist_weight: 5.0\n depth_gaussian_sigma: 1.0\n num_grasp_samples: 1000\n\n max_dist_from_center: 260\n max_num_samples: 10000\n\n # metric params\n metric:\n type: gqcnn\n gqcnn_model: models/GQCNN-4.0-SUCTION\n\n crop_height: 96\n crop_width: 96\n\n # visualization\n vis:\n grasp_sampling : 0\n tf_images: 0\n plane: 0\n grasp_candidates: 0\n elite_grasps: 0\n grasp_ranking: 0\n grasp_plan: 0\n grasp_affordance_map: 0\n final_grasp: 1\n\n vmin: 0.0\n vmax: 1.0\n\n k: 25\n\n# image proc params\ninpaint_rescale_factor: 0.5\n\n# detection params\ndetection:\n type: point_cloud_box\n\n foreground_mask_tolerance: 60\n min_pt:\n - 0.205\n - -0.3\n - 0.01\n\n max_pt:\n - 0.65\n - 0.3\n - 0.15\n\n selection_policy: min\n focus: 0\n\n min_contour_area: 250.0\n max_contour_area: 1000000.0\n min_box_area: 250.0\n max_box_area: 1000000.0\n box_padding_px: 15\n\n rescale_factor: 1.0\n interpolation: bilinear\n depth_grad_thresh: 10.0\n contour_dist_thresh: 2.5\n\n point_cloud_mask_only: 1\n\n image_width: 640\n image_height: 480\n\n filter_dim: 1\n" }, { "path": "cfg/examples/ros/fc_gqcnn_pj.yaml", "content": "!include ../fc_gqcnn_pj.yaml\n\n# ROS-specific visualization\nvis:\n rgbd_state: 0\n cropped_rgbd_image: 0\n color_image: 0\n depth_image: 0\n segmask: 0\n\n" }, { "path": "cfg/examples/ros/fc_gqcnn_suction.yaml", "content": "!include ../fc_gqcnn_suction.yaml\n\n# ROS-specific visualization\nvis:\n rgbd_state: 0\n cropped_rgbd_image: 0\n color_image: 0\n depth_image: 0\n segmask: 0\n\n" }, { "path": "cfg/examples/ros/gqcnn_pj.yaml", "content": "!include ../gqcnn_pj.yaml\n\n# ROS-specific visualization\nvis:\n rgbd_state: 0\n cropped_rgbd_image: 0\n color_image: 0\n depth_image: 0\n segmask: 0\n\n" }, { "path": "cfg/examples/ros/gqcnn_suction.yaml", "content": "!include ../gqcnn_suction.yaml\n\n# ROS-specific visualization\nvis:\n rgbd_state: 0\n cropped_rgbd_image: 0\n color_image: 0\n depth_image: 0\n segmask: 0\n\n" }, { "path": "cfg/finetune.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 10 # number of epochs to train for\neval_frequency: 2.5 # how often to get validation error (in epochs)\nsave_frequency: 2.5 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params \ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\noptimize_base_layers: 0\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n base_model:\n output_layer: fc4\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 8\n pool_size: 2\n pool_stride: 2 \n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 64\n pose_stream:\n pc1:\n type: pc\n out_size: 4\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 64\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/finetune_dex-net_4.0_pj.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10.0 # how often to get validation error (in epochs)\nsave_frequency: 10.0 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params \ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\noptimize_base_layers: 0\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n base_model:\n output_layer: fc4\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 16\n pool_size: 1\n pool_stride: 1 \n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 128\n pose_stream:\n pc1:\n type: pc\n out_size: 16\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 128\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/finetune_dex-net_4.0_suction.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10.0 # how often to get validation error (in epochs)\nsave_frequency: 10.0 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params \ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\noptimize_base_layers: 0\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: suction\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n base_model:\n output_layer: fc3\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 16\n pool_size: 1\n pool_stride: 1 \n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 128\n pose_stream:\n pc1:\n type: pc\n out_size: 16\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 128\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/finetune_example_pj.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 20 # number of epochs to train for\neval_frequency: 10.0 # how often to get validation error (in epochs)\nsave_frequency: 10.0 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params \ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.33 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\noptimize_base_layers: 0\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n base_model:\n output_layer: conv2_2\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 16\n pool_size: 1\n pool_stride: 1 \n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 64\n pose_stream:\n pc1:\n type: pc\n out_size: 16\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 64\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/finetune_example_suction.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 20 # number of epochs to train for\neval_frequency: 10.0 # how often to get validation error (in epochs)\nsave_frequency: 10.0 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params \ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.33 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\noptimize_base_layers: 0\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: suction\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n base_model:\n output_layer: conv2_2\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 16\n pool_size: 1\n pool_stride: 1 \n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 64\n pose_stream:\n pc1:\n type: pc\n out_size: 16\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 64\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/hyperparam_search/train_dex-net_4.0_fc_suction_hyperparam_search.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params \ntrain_pct: 0.8 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 10000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: 0\ndebug_num_files: 1000000000\nseed: 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: 0\n seed: 24098\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: suction\n\n # method by which to integrate depth into the network\n input_depth_mode: im_only\n \n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: ['anchor_filt_dim', 6, 9, 9]\n num_filt: ['anchor_num_filts', 8, 12, 16, 20]\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: ['anchor_filt_dim', 3, 5, 5]\n num_filt: ['anchor_num_filts', 8, 12, 16, 20]\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: ['anchor_filt_dim', 3, 5, 3]\n num_filt: ['anchor_num_filts', 8, 12, 16, 20]\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: ['anchor_filt_dim', 3, 5, 3]\n num_filt: ['anchor_num_filts', 8, 12, 16, 20]\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: ['anchor_fc_out_size', 32, 64, 128]\n fc4:\n type: fc\n out_size: ['anchor_fc_out_size', 32, 64, 128]\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/hyperparam_search/train_hyperparam_search.yaml", "content": "# Example hyper-parameter search config corresponding to cfg/train.yaml that\n# grid-searches through number of training epochs, base learning rate, and\n# conv1_1 filter sizes.\n\n# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: [25, 50, 75] # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: [0.001, 0.01, 0.05, 0.1]\ndecay_step_multiplier: 1.0 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: 0\ndebug_num_files: 1000000000\nseed: 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: 0\n seed: 24098\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: [3, 5, 9]\n num_filt: 8\n pool_size: 2\n pool_stride: 2 \n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 64\n pose_stream:\n pc1:\n type: pc\n out_size: 4\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 64\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/tools/analyze_gqcnn_performance.yaml", "content": "log_rate: 10\nfont_size: 15\nline_width: 4\ndpi: 100\nnum_bins: 100\nnum_vis: 10\n" }, { "path": "cfg/tools/run_policy.yaml", "content": "# policy params\npolicy:\n # optimization params\n num_seed_samples: 250\n num_gmm_samples: 100\n num_iters: 3\n gmm_refit_p: 0.125\n gmm_component_frac: 0.8\n gmm_reg_covar: 0.01\n max_resamples_per_iteration: 10\n\n # general params\n deterministic: 0\n gripper: yumi_metal_spline\n gripper_width: 0.05\n max_approach_angle: 60\n logging_dir: logs/debug\n\n # sampling params\n sampling:\n # type\n type: antipodal_depth\n\n # antipodality\n friction_coef: 0.8\n depth_grad_thresh: 0.002\n depth_grad_gaussian_sigma: 0.5\n min_num_edge_pixels: 25\n downsample_rate: 2\n max_rejection_samples: 4000\n\n # distance\n max_dist_from_center: 100000\n min_dist_from_boundary: 45\n min_grasp_dist: 1.0\n angle_dist_weight: 5.0\n\n # depth sampling\n depth_samples_per_grasp: 1\n depth_sample_win_height: 1\n depth_sample_win_width: 1\n\n depth_sampling_mode: uniform\n min_depth_offset: 0.015\n max_depth_offset: 0.04\n\n # metric params\n metric:\n type: gqcnn\n\n gqcnn_model: /mnt/data/diskstation/models/icra2019/gqcnn_mini_dexnet/\n\n crop_height: 96\n crop_width: 96\n\n # filter params\n filter_collisions: 1\n filter_ik: 1\n filter_unreachable: 0\n max_grasps_filter: 10\n\n collision_free_filter:\n approach_dist: 0.075\n rescale_factor: 0.125\n ik_filter:\n approach_dist: 0.075\n traj_len: 0\n group_name: left_jaws\n ik_timeout: 0.1\n reachability_filter:\n unreachable_pose_dir: /mnt/data/diskstation/unreachable_poses/left/\n min_rot_dist: 0.1\n min_trans_dist: 0.01\n\n # visualization\n vis:\n input_images: 0\n grasp_sampling : 0\n tf_images: 0\n grasp_candidates: 0\n elite_grasps: 0\n grasp_ranking: 0\n grasp_plan: 0\n seg_point_cloud: 0\n filtered_point_cloud: 0\n final_grasp: 1\n\n grasp_scale: 1\n\n vmin: 0.6\n vmax: 0.9\n\n k: 25\n" }, { "path": "cfg/train.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 1.0 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 8\n pool_size: 2\n pool_stride: 2 \n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 64\n pose_stream:\n pc1:\n type: pc\n out_size: 4\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 64\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_dex-net_2.0.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 5 # how often to get validation error (in epochs)\nsave_frequency: 5 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.8 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.66 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: depth_ims_tf_table\npose_field_name: hand_poses\n\n# label params\ntarget_metric_name: robust_ferrari_canny # name of the field to use for the labels\nmetric_thresh: 0.002 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 10000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 1\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 1\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 32\n im_width: 32\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: legacy_parallel_jaw\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 7\n num_filt: 64\n pool_size: 1\n pool_stride: 1\n pad: SAME\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 64\n pool_size: 2\n pool_stride: 2\n pad: SAME\n norm: 1\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 3\n num_filt: 64\n pool_size: 1\n pool_stride: 1\n pad: SAME\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 3\n num_filt: 64\n pool_size: 2\n pool_stride: 2\n pad: SAME\n norm: 1\n norm_type: local_response\n fc3:\n type: fc\n out_size: 1024\n pose_stream:\n pc1:\n type: pc\n out_size: 16\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 1024\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_dex-net_3.0.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 25 # number of epochs to train for\neval_frequency: 5 # how often to get validation error (in epochs)\nsave_frequency: 5 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params \ntrain_pct: 0.8 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.66 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: depth_ims_tf_table\npose_field_name: hand_poses\n\n# label params\ntarget_metric_name: robust_suction_wrench_resistance # name of the field to use for the labels\nmetric_thresh: 0.2 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 10000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 1\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 1\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 32\n im_width: 32\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: legacy_suction\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 7\n num_filt: 64\n pool_size: 1\n pool_stride: 1\n pad: SAME\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 64\n pool_size: 2\n pool_stride: 2\n pad: SAME\n norm: 1\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 3\n num_filt: 64\n pool_size: 1\n pool_stride: 1\n pad: SAME\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 3\n num_filt: 64\n pool_size: 2\n pool_stride: 2\n pad: SAME\n norm: 1\n norm_type: local_response\n fc3:\n type: fc\n out_size: 1024\n pose_stream:\n pc1:\n type: pc\n out_size: 128\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 1024\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_dex-net_4.0_fc_pj.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.8 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 10000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 0\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # method by which to integrate depth into the network\n input_depth_mode: im_depth_sub\n \n # used for training with multiple angular predictions\n angular_bins: 16\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 128\n fc4:\n type: fc\n out_size: 128\n fc5:\n type: fc\n out_size: 32\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_dex-net_4.0_fc_suction.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.8 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 10000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: suction\n\n # method by which to integrate depth into the network\n input_depth_mode: im_only\n \n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 128\n fc4:\n type: fc\n out_size: 128\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_dex-net_4.0_pj.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.8 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 10000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 128\n pose_stream:\n pc1:\n type: pc\n out_size: 16\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 128\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_dex-net_4.0_suction.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.8 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 10000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: suction\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 8\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 64\n pose_stream:\n pc1:\n type: pc\n out_size: 16\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 64\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_example_pj.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 100 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 1.0 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.98\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n \n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 8\n pool_size: 2\n pool_stride: 2 \n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 64\n pose_stream:\n pc1:\n type: pc\n out_size: 4\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 64\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_example_suction.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.9 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 1.0 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 1000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 1\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: suction\n\n # method by which to integrate depth into the network\n input_depth_mode: pose_stream\n\n # used for training with multiple angular predictions\n angular_bins: 0\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 8\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 3\n num_filt: 8\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 64\n pose_stream:\n pc1:\n type: pc\n out_size: 16\n pc2:\n type: pc\n out_size: 0\n merge_stream:\n fc4:\n type: fc_merge\n out_size: 64\n fc5:\n type: fc\n out_size: 2\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "cfg/train_fc.yaml", "content": "# general optimization params\ntrain_batch_size: 64\nval_batch_size: &val_batch_size 64\n\n# logging params\nnum_epochs: 50 # number of epochs to train for\neval_frequency: 10 # how often to get validation error (in epochs)\nsave_frequency: 10 # how often to save output (in epochs)\nvis_frequency: 10000 # how often to visualize filters (in epochs)\nlog_frequency: 1 # how often to log output (in steps)\n\n# train / val split params\ntrain_pct: 0.8 # percentage of the data to use for training vs validation\ntotal_pct: 1.0 # percentage of all the files to use\neval_total_train_error: 0 # whether or not to evaluate the total training error on each validataion\nmax_files_eval: 1000 # the number of validation files to use in each eval\n\n# optimization params\nloss: sparse\noptimizer: momentum\ntrain_l2_regularizer: 0.0005\nbase_lr: 0.01\ndecay_step_multiplier: 0.5 # number of times to go through training datapoints before stepping down decay rate (in epochs)\ndecay_rate: 0.95\nmomentum_rate: 0.9\nmax_training_examples_per_load: 128\ndrop_rate: 0.0\nmax_global_grad_norm: 100000000000\n\n# input params\ntraining_mode: classification\nimage_field_name: tf_depth_ims\npose_field_name: grasps\n\n# label params\ntarget_metric_name: grasp_metrics # name of the field to use for the labels\nmetric_thresh: 0.5 # threshold for positive examples (label = 1 if grasp_metric > metric_thresh)\n\n# preproc params\nnum_random_files: 10000 # the number of random files to compute dataset statistics in preprocessing (lower speeds initialization)\npreproc_log_frequency: 100 # how often to log preprocessing (in steps)\n\n# denoising / synthetic data params\nmultiplicative_denoising: 0\ngamma_shape: 1000.00\n\nsymmetrize: 0\n\ngaussian_process_denoising: 0\ngaussian_process_rate: 0.5\ngaussian_process_scaling_factor: 4.0\ngaussian_process_sigma: 0.005\n\n# tensorboard\ntensorboard_port: 6006\n\n# debugging params\ndebug: &debug 0\ndebug_num_files: 10 # speeds up initialization\nseed: &seed 24098\n\n### GQCNN CONFIG ###\ngqcnn:\n # basic data metrics\n im_height: 96\n im_width: 96\n im_channels: 1\n debug: *debug\n seed: *seed\n\n # needs to match input data mode that was used for training, determines the pose dimensions for the network\n gripper_mode: parallel_jaw\n\n # method by which to integrate depth into the network\n input_depth_mode: im_depth_sub\n \n # used for training with multiple angular predictions\n angular_bins: 16\n\n # prediction batch size, in training this will be overriden by the val_batch_size in the optimizer's config file\n batch_size: *val_batch_size\n\n # architecture\n architecture:\n im_stream:\n conv1_1:\n type: conv\n filt_dim: 9\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv1_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_1:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 1\n pool_stride: 1\n pad: VALID\n norm: 0\n norm_type: local_response\n conv2_2:\n type: conv\n filt_dim: 5\n num_filt: 16\n pool_size: 2\n pool_stride: 2\n pad: VALID\n norm: 0\n norm_type: local_response\n fc3:\n type: fc\n out_size: 128\n fc4:\n type: fc\n out_size: 128\n fc5:\n type: fc\n out_size: 32\n\n # architecture normalization constants\n radius: 2\n alpha: 2.0e-05\n beta: 0.75\n bias: 1.0\n\n # leaky relu coefficient\n relu_coeff: 0.0\n" }, { "path": "ci/travis/format.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n# Script for YAPF formatting. Adapted from https://github.com/ray-project/ray/blob/master/ci/travis/format.sh.\n\nYAPF_FLAGS=(\n '--style' \".style.yapf\"\n '--recursive'\n '--parallel'\n)\n\nYAPF_EXCLUDES=()\n\n# Format specified files\nformat() {\n yapf --in-place \"${YAPF_FLAGS[@]}\" -- \"$@\"\n}\n\n# Format all files, and print the diff to `stdout` for Travis.\nformat_all() {\n yapf --diff \"${YAPF_FLAGS[@]}\" \"${YAPF_EXCLUDES[@]}\" .\n}\n\n# This flag formats individual files. `--files` *must* be the first command line\n# arg to use this option.\nif [[ \"$1\" == '--files' ]]; then\n format \"${@:2}\"\n # If `--all` is passed, then any further arguments are ignored and the\n # entire Python directory is formatted.\nelif [[ \"$1\" == '--all' ]]; then\n format_all\nfi\n" }, { "path": "data/calib/phoxi/phoxi.intr", "content": "{\"_cy\": 191.75, \"_cx\": 255.5, \"_fy\": 552.5, \"_height\": 386, \"_fx\": 552.5, \"_width\": 516, \"_skew\": 0.0, \"_K\": 0, \"_frame\": \"phoxi\"}" }, { "path": "data/calib/phoxi/phoxi_to_world.tf", "content": "phoxi\nworld\n0.385146 -0.121589 0.808145\n0.005659 -0.999983 0.001249\n-0.989208 -0.005415 0.146420\n-0.146410 -0.002064 -0.989222\n" }, { "path": "data/calib/primesense/primesense.intr", "content": "{\"_cy\": 239.5, \"_cx\": 319.5, \"_fy\": 525.0, \"_height\": 480, \"_fx\": 525.0, \"_width\": 640, \"_skew\": 0.0, \"_K\": 0, \"_frame\": \"primesense_overhead\"}" }, { "path": "data/calib/primesense/primesense.tf", "content": "primesense_overhead\nworld\n0.214280 0.004186 0.872913\n0.021188 -0.975273 0.219984\n-0.999765 -0.021698 0.000098\n0.004678 -0.219934 -0.975504\n" }, { "path": "docker/cpu/Dockerfile", "content": "FROM ubuntu:xenial\n\nMAINTAINER Vishal Satish \n\n# Args.\n# Must be an absolute path.\nARG work_dir=/root/Workspace\n\n# Install `apt-get` deps.\nRUN apt-get update && apt-get install -y \\\n build-essential \\\n python3 \\\n python3-dev \\\n python3-tk \\\n python-opengl \\\n curl \\\n libsm6 \\\n libxext6 \\\n libglib2.0-0 \\\n libxrender1 \\\n wget \\\n unzip\n\n# Install libspatialindex (required for latest rtree).\nRUN curl -L http://download.osgeo.org/libspatialindex/spatialindex-src-1.8.5.tar.gz | tar xz && \\\n cd spatialindex-src-1.8.5 && \\\n ./configure && \\\n make && \\\n make install && \\\n ldconfig && \\\n cd ..\n\n# Install pip (`apt-get install python-pip` causes trouble w/ networkx).\nRUN curl -O https://bootstrap.pypa.io/get-pip.py && \\\n\tpython3 get-pip.py && \\\n\trm get-pip.py\n\n# Required for easy_install to find right skimage version for Python 3.5.\nRUN pip3 install -U setuptools\n\n# Make working directory.\nWORKDIR ${work_dir}\n\n# Copy the library.\nADD docker/gqcnn.tar .\n\n# This is because `python setup.py develop` skips `install_requires` (I think).\nRUN python3 -m pip install -r gqcnn/requirements/cpu_requirements.txt\n\n# Install the library in editable mode because it's more versatile (in case we want to develop or if users want to modify things)\n# Keep the egg outside of the library in site-packages in case we want to mount the library (overwriting it) for development with docker\nENV PYTHONPATH ${work_dir}/gqcnn\nWORKDIR /usr/local/lib/python3.5/site-packages/\nRUN python3 ${work_dir}/gqcnn/setup.py develop --docker\n\n# Move to the top-level gqcnn package dir.\nWORKDIR ${work_dir}/gqcnn\n" }, { "path": "docker/gpu/Dockerfile", "content": "FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04\n\nMAINTAINER Vishal Satish \n\n# Args\n# `work_dir` must be an absolute path.\nARG work_dir=/root/Workspace\n\n# Install `apt-get` deps.\nRUN apt-get update && apt-get install -y \\\n build-essential \\\n python3 \\\n python3-dev \\\n python3-tk \\\n python-opengl \\\n curl \\\n libsm6 \\\n libxext6 \\\n libglib2.0-0 \\\n libxrender1 \\\n wget \\\n unzip\n\n# Install libspatialindex (required for latest rtree).\nRUN curl -L http://download.osgeo.org/libspatialindex/spatialindex-src-1.8.5.tar.gz | tar xz && \\\n cd spatialindex-src-1.8.5 && \\\n ./configure && \\\n make && \\\n make install && \\\n ldconfig && \\\n cd ..\n\n# Install pip (`apt-get install python-pip` causes trouble w/ networkx).\nRUN curl -O https://bootstrap.pypa.io/get-pip.py && \\\n\tpython3 get-pip.py && \\\n\trm get-pip.py\n\n# Required for easy_install to find right skimage version for Python 3.5.\nRUN pip3 install -U setuptools\n\n# Make working directory.\nWORKDIR ${work_dir}\n\n# Copy the library.\nADD docker/gqcnn.tar .\n\n# This is because `python setup.py develop` skips install_requires (I think) and also because we want to explicitly use the GPU requirements.\nRUN python3 -m pip install -r gqcnn/requirements/gpu_requirements.txt\n\n# Install the library in editable mode because it's more versatile (in case we want to develop or if users want to modify things)\n# Keep the egg outside of the library in site-packages in case we want to mount the library (overwriting it) for development with docker\nENV PYTHONPATH ${work_dir}/gqcnn\nWORKDIR /usr/local/lib/python3.5/site-packages/\nRUN python3 ${work_dir}/gqcnn/setup.py develop --docker\n\n# Move to the base library dir\nWORKDIR ${work_dir}/gqcnn\n" }, { "path": "docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nSPHINXPROJ = GQCNN\nSOURCEDIR = source\nBUILDDIR = build\nGH_PAGES_SOURCES = docs examples gqcnn tools\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n.PHONY: clean\nclean:\n\trm -rf $(BUILDDIR)/*\n\n.PHONY: gh-pages\ngh-pages:\n\tgit checkout gh-pages && \\\n\tcd .. && \\\n\tgit rm -rf . && git clean -fxd && \\\n\tgit checkout master $(GH_PAGES_SOURCES) && \\\n\tgit reset HEAD && \\\n\tcd docs && \\\n\tmake html && \\\n\tcd .. && \\\n\tmv -fv docs/build/html/* ./ && \\\n\ttouch .nojekyll && \\\n\trm -rf $(GH_PAGES_SOURCES) && \\\n\tgit add -A && \\\n\tgit commit -m \"Generated gh-pages for `git log master -1 --pretty=short --abbrev-commit`\" && \\\n\tgit push origin --delete gh-pages && \\\n\tgit push origin gh-pages ; \\\n\tgit checkout master\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n" }, { "path": "docs/gh_deploy.sh", "content": "#!/bin/sh\nmake gh-pages\ncd ../docs\n" }, { "path": "docs/make.bat", "content": "@ECHO OFF\r\n\r\npushd %~dp0\r\n\r\nREM Command file for Sphinx documentation\r\n\r\nif \"%SPHINXBUILD%\" == \"\" (\r\n\tset SPHINXBUILD=sphinx-build\r\n)\r\nset SOURCEDIR=source\r\nset BUILDDIR=build\r\nset SPHINXPROJ=GQCNN\r\n\r\nif \"%1\" == \"\" goto help\r\n\r\n%SPHINXBUILD% >NUL 2>NUL\r\nif errorlevel 9009 (\r\n\techo.\r\n\techo.The 'sphinx-build' command was not found. Make sure you have Sphinx\r\n\techo.installed, then set the SPHINXBUILD environment variable to point\r\n\techo.to the full path of the 'sphinx-build' executable. Alternatively you\r\n\techo.may add the Sphinx directory to PATH.\r\n\techo.\r\n\techo.If you don't have Sphinx installed, grab it from\r\n\techo.http://sphinx-doc.org/\r\n\texit /b 1\r\n)\r\n\r\n%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%\r\ngoto end\r\n\r\n:help\r\n%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%\r\n\r\n:end\r\npopd\r\n" }, { "path": "docs/source/api/analysis.rst", "content": "Analysis\n========\n\nGQCNNAnalyzer\n~~~~~~~~~~~~~\nA tool for analyzing trained GQ-CNNs. Calculates statistics such as training/valiation errors and losses. Also plots Precision-Recall Curve and ROC, and saves sample TP/TN/FP/FN training/validation examples. \n\n.. autoclass:: gqcnn.GQCNNAnalyzer\n\n" }, { "path": "docs/source/api/gqcnn.rst", "content": "GQ-CNN\n======\n\nGQ-CNN and FC-GQ-CNN classes are **never accessed directly**, but through a lightweight factory function that returns the corresponding class depending on the specified backend. ::\n\n $ from gqcnn import get_gqcnn_model\n $\n $ backend = 'tf'\n $ my_gqcnn = get_gqcnn_model(backend)()\n\n.. autofunction:: gqcnn.get_gqcnn_model\n\n.. autofunction:: gqcnn.get_fc_gqcnn_model\n\nGQCNNTF\n~~~~~~~\n\nTensorflow implementation of GQ-CNN model.\n\n.. autoclass:: gqcnn.model.tf.GQCNNTF\n :exclude-members: init_mean_and_std,\n set_base_network,\n init_weights_file,\n initialize_network,\n set_batch_size,\n set_im_mean,\n get_im_mean,\n set_im_std,\n get_im_std,\n set_pose_mean,\n get_pose_mean,\n set_pose_std,\n get_pose_std,\n set_im_depth_sub_mean,\n set_im_depth_sub_std,\n add_softmax_to_output,\n add_sigmoid_to_output,\n update_batch_size,\n \nFCGQCNNTF\n~~~~~~~~~\n\nTensorflow implementation of FC-GQ-CNN model.\n\n.. autoclass:: gqcnn.model.tf.FCGQCNNTF\n :exclude-members: __init__\n" }, { "path": "docs/source/api/policies.rst", "content": "Policies\n========\n\nAll GQ-CNN grasping policies are child classes of the base :ref:`GraspingPolicy` class that implements `__call__()`, which operates on :ref:`RgbdImageStates ` and returns a :ref:`GraspAction`. :: \n\n $ from gqcnn import RgbdImageState, CrossEntropyRobustGraspingPolicy\n $\n $ im = RgbdImageState.load()\n $ my_policy = CrossEntropyRobustGraspingPolicy()\n $\n $ my_grasp_action = my_policy(im)\n\nPrimary Policies\n~~~~~~~~~~~~~~~~\n\nCrossEntropyRobustGraspingPolicy\n--------------------------------\nAn implementation of the `Cross Entropy Method (CEM)`_ used in `Dex-Net 2.0`_, `Dex-Net 2.1`_, `Dex-Net 3.0`_, and `Dex-Net 4.0`_ to iteratively locate the best grasp.\n\n.. autoclass:: gqcnn.CrossEntropyRobustGraspingPolicy\n\nFullyConvolutionalGraspingPolicyParallelJaw\n-------------------------------------------\nAn implementation of the `FC-GQ-CNN`_ parallel jaw policy that uses dense, parallelized fully convolutional networks.\n\n.. autoclass:: gqcnn.FullyConvolutionalGraspingPolicyParallelJaw \n\nFullyConvolutionalGraspingPolicySuction\n---------------------------------------\nAn implementation of the `FC-GQ-CNN`_ suction policy that uses dense, parallelized fully convolutional networks.\n\n.. autoclass:: gqcnn.FullyConvolutionalGraspingPolicySuction\n\nGrasps and Image Wrappers\n~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. _RgbdImageState:\n\nRgbdImageState\n--------------\nA wrapper for states containing an RGBD (RGB + Depth) image, camera intrinisics, and segmentation masks.\n\n.. autoclass:: gqcnn.RgbdImageState\n\n.. _GraspAction:\n\nGraspAction\n-----------\nA wrapper for 2D grasp actions such as :ref:`Grasp2D` or :ref:`SuctionPoint2D`.\n\n.. autoclass:: gqcnn.grasping.policy.policy.GraspAction\n\n.. _Grasp2D:\n\nGrasp2D\n-------\nA wrapper for 2D parallel jaw grasps.\n\n.. autoclass:: gqcnn.grasping.grasp.Grasp2D\n\n.. _SuctionPoint2D:\n\nSuctionPoint2D\n--------------\nA wrapper for 2D suction grasps.\n\n.. autoclass:: gqcnn.grasping.grasp.SuctionPoint2D\n\nMiscellaneous and Parent Policies\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nPolicy\n------\n\n.. autoclass:: gqcnn.grasping.policy.policy.Policy\n\n\n.. _GraspingPolicy:\n\nGraspingPolicy\n--------------\n\n.. autoclass:: gqcnn.grasping.policy.policy.GraspingPolicy\n\nFullyConvolutionalGraspingPolicy\n--------------------------------\n.. autoclass:: gqcnn.grasping.policy.fc_policy.FullyConvolutionalGraspingPolicy\n\nRobustGraspingPolicy\n--------------------\n\n.. autoclass:: gqcnn.RobustGraspingPolicy\n\nUniformRandomGraspingPolicy\n---------------------------\n\n.. autoclass:: gqcnn.UniformRandomGraspingPolicy\n\n.. _Cross Entropy Method (CEM): https://en.wikipedia.org/wiki/Cross-entropy_method\n.. _Dex-Net 2.0: https://berkeleyautomation.github.io/dex-net/#dexnet_2\n.. _Dex-Net 2.1: https://berkeleyautomation.github.io/dex-net/#dexnet_21\n.. _Dex-Net 3.0: https://berkeleyautomation.github.io/dex-net/#dexnet_3\n.. _Dex-Net 4.0: https://berkeleyautomation.github.io/dex-net/#dexnet_4\n.. _FC-GQ-CNN: https://berkeleyautomation.github.io/fcgqcnn\n\n" }, { "path": "docs/source/api/training.rst", "content": "Training\n========\n\nGQ-CNN training classes are **never accessed directly**, but through a lightweight factory function that returns the corresponding class depending on the specified backend. ::\n\n $ from gqcnn import get_gqcnn_trainer\n $\n $ backend = 'tf'\n $ my_trainer = get_gqcnn_trainer(backend)()\n\n.. autofunction:: gqcnn.get_gqcnn_trainer\n\nGQCNNTrainerTF\n~~~~~~~~~~~~~~\n\n.. autoclass:: gqcnn.training.tf.GQCNNTrainerTF\n\n" }, { "path": "docs/source/benchmarks/benchmarks.rst", "content": "Dex-Net 2.0\n~~~~~~~~~~~\nBelow are the highest classification accuracies achieved on the `Dex-Net 2.0`_ dataset on a randomized 80-20 train-validation split using various splitting rules:\n\n.. image:: ../images/gqcnn_leaderboard.png\n :width: 100%\n\nThe current leader is a ConvNet submitted by nomagic.ai. `GQ` is our best GQ-CNN for `Dex-Net 2.0`_.\t \n\nWe believe grasping performance on the physical robot can be improved if these validation error rates can be further reduced by modifications to the network architecture and optimization.\nIf you achieve superior numbers on a randomized validation set, please email Jeff Mahler (jmahler@berkeley.edu) with the subject \"Dex-Net 2.0 Benchmark Submission\" and we will consider testing on our ABB YuMi.\n\n.. _Dex-Net 2.0: https://berkeleyautomation.github.io/dex-net/#dexnet_2\n\n" }, { "path": "docs/source/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# core documentation build configuration file, created by\n# sphinx-quickstart on Sun Oct 16 14:33:48 2016.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\nimport sys\nimport os\nimport sphinx_rtd_theme\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\nsys.path.insert(\n 0, os.path.join(os.path.abspath(os.path.dirname(__name__)), \"../../\"))\nsys.path.insert(\n 0,\n os.path.join(os.path.abspath(os.path.dirname(__name__)), \"../../examples\"))\nsys.path.insert(\n 0, os.path.join(os.path.abspath(os.path.dirname(__name__)), \"../../tools\"))\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#needs_sphinx = \"1.0\"\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named \"sphinx.ext.*\") or your custom\n# ones.\nextensions = [\"sphinx.ext.autodoc\", \"sphinxcontrib.napoleon\"]\nautoclass_content = \"class\"\nautodoc_member_order = \"bysource\"\nautodoc_default_flags = [\"members\", \"show-inheritance\"]\nnapoleon_include_special_with_doc = True\nnapoleon_include_init_with_doc = True\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = [\"_templates\"]\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n# source_suffix = [\".rst\", \".md\"]\nsource_suffix = \".rst\"\n\n# The encoding of source files.\n#source_encoding = \"utf-8-sig\"\n\n# The master toctree document.\nmaster_doc = \"index\"\n\n# General information about the project.\nproject = u\"GQCNN\"\ncopyright = u\"2019, The Regents of the University of California\"\nauthor = u\"Vishal Satish, Jeff Mahler\"\n\n# The version info for the project you\"re documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = u\"1.1\"\n# The full version, including alpha/beta/rc tags.\nrelease = u\"1.1.0\"\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n#today = \"\"\n# Else, today_fmt is used as the format for a strftime call.\n#today_fmt = \"%B %d, %Y\"\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\nexclude_patterns = []\n\n# The reST default role (used for this markup: `text`) to use for all\n# documents.\n#default_role = None\n\n# If true, \"()\" will be appended to :func: etc. cross-reference text.\n#add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n#add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n#show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = \"sphinx\"\n\n# A list of ignored prefixes for module index sorting.\n#modindex_common_prefix = []\n\n# If true, keep warnings as \"system message\" paragraphs in the built documents.\n#keep_warnings = False\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\nhtml_theme = \"sphinx_rtd_theme\"\nhtml_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\n#html_theme_path = []\n\n# The name for this set of Sphinx documents. If None, it defaults to\n# \" v documentation\".\n#html_title = None\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n#html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\n#html_logo = None\n\n# The name of an image file (relative to this directory) to use as a favicon of\n# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\n#html_favicon = None\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\n# html_static_path = [\"_static\"]\n\n# Add any extra paths that contain custom files (such as robots.txt or\n# .htaccess) here, relative to this directory. These files are copied\n# directly to the root of the documentation.\n#html_extra_path = []\n\n# If not \"\", a \"Last updated on:\" timestamp is inserted at every page bottom,\n# using the given strftime format.\n#html_last_updated_fmt = \"%b %d, %Y\"\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n#html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\n#html_sidebars = {}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n#html_additional_pages = {}\n\n# If false, no module index is generated.\n#html_domain_indices = True\n\n# If false, no index is generated.\n#html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n#html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n#html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\n#html_show_sphinx = True\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\n#html_show_copyright = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n#html_use_opensearch = \"\"\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n#html_file_suffix = None\n\n# Language to be used for generating the HTML full-text search index.\n# Sphinx supports the following languages:\n# \"da\", \"de\", \"en\", \"es\", \"fi\", \"fr\", \"hu\", \"it\", \"ja\"\n# \"nl\", \"no\", \"pt\", \"ro\", \"ru\", \"sv\", \"tr\"\n#html_search_language = \"en\"\n\n# A dictionary with options for the search language support, empty by default.\n# Now only \"ja\" uses this config value\n#html_search_options = {\"type\": \"default\"}\n\n# The name of a javascript file (relative to the configuration directory) that\n# implements a search results scorer. If empty, the default will be used.\n#html_search_scorer = \"scorer.js\"\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = \"GQCNNdoc\"\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n # The paper size (\"letterpaper\" or \"a4paper\").\n #\"papersize\": \"letterpaper\",\n\n # The font size (\"10pt\", \"11pt\" or \"12pt\").\n #\"pointsize\": \"10pt\",\n\n # Additional stuff for the LaTeX preamble.\n #\"preamble\": \"\",\n\n # Latex figure (float) alignment\n #\"figure_align\": \"htbp\",\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, \"GQCNN.tex\", u\"GQCNN Documentation\",\n u\"Jeff Mahler, Vishal Satish\", \"manual\"),\n]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n#latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n#latex_use_parts = False\n\n# If true, show page references after internal links.\n#latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\n#latex_show_urls = False\n\n# Documents to append as an appendix to all manuals.\n#latex_appendices = []\n\n# If false, no module index is generated.\n#latex_domain_indices = True\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [(master_doc, \"GQCNN\", u\"GQCNN Documentation\", [author], 1)]\n\n# If true, show URL addresses after external links.\n#man_show_urls = False\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, \"GQCNN\", u\"GQCNN Documentation\", author, \"GQCNN\",\n \"One line description of project.\", \"Miscellaneous\"),\n]\n\n# Documents to append as an appendix to all manuals.\n#texinfo_appendices = []\n\n# If false, no module index is generated.\n#texinfo_domain_indices = True\n\n# How to display URL addresses: \"footnote\", \"no\", or \"inline\".\n#texinfo_show_urls = \"footnote\"\n\n# If true, do not generate a @detailmenu in the \"Top\" node\"s menu.\n#texinfo_no_detailmenu = False\n" }, { "path": "docs/source/index.rst", "content": ".. GQCNN documentation master file, created by\n sphinx-quickstart on Thu May 4 16:09:26 2017.\n You can adapt this file completely to your liking, but it should at least\n contain the root `toctree` directive.\n\nBerkeley AUTOLAB's GQCNN Package\n================================\n\nOverview\n--------\nThe `gqcnn` package is a Python API for training and deploying `Grasp Quality Convolutional Neural Networks (GQ-CNNs)`_ for grasp planning using training datasets from the `Dexterity Network (Dex-Net)`_, developed by the `Berkeley AUTOLAB`_ and introduced in the `Dex-Net 2.0 paper`_.\n\n.. note:: We're excited to announce **version 1.0**, which brings the GQ-CNN package up to date with recent research in `Dex-Net`_.\n Version 1.0 introduces support for:\n\n #. **Dex-Net 4.0:** Composite policies that decide whether to use a suction cup or parallel-jaw gripper.\n #. **Fully Convolutional GQ-CNNs:** Fully convolutional architectures that efficiently evaluate millions of grasps faster than prior GQ-CNNs.\n\nLinks\n-----\n* `Source Code`_\n* `Datasets`_\n* `Pretrained Models`_\n* `Dex-Net Website`_\n* `UC Berkeley AUTOLAB`_\n\n.. _Source Code: https://github.com/BerkeleyAutomation/gqcnn\n.. _Datasets: http://bit.ly/2rIM7Jk\n.. _Pretrained Models: http://bit.ly/2tAFMko\n.. _Dex-Net Website: https://berkeleyautomation.github.io/dex-net\n.. _UC Berkeley AUTOLAB: http://autolab.berkeley.edu\n\n.. image:: images/gqcnn.png\n :width: 100 %\n\nProject Goals\n-------------\nThe goals of this project are to facilitate:\n\n#. **Replicability** of GQ-CNN training and deployment from:\n #. The latest `Dex-Net 4.0`_ results.\n #. Older results such as `Dex-Net 2.0`_, `Dex-Net 2.1`_ and `Dex-Net 3.0`_.\n #. Experimental results such as `FC-GQ-CNN`_.\n#. **Research extensions** on novel GQ-CNN architectures that have higher performance on `Dex-Net 4.0`_ training datasets.\n\n.. _Dex-Net 2.0: https://berkeleyautomation.github.io/dex-net/#dexnet_2\n.. _Dex-Net 2.1: https://berkeleyautomation.github.io/dex-net/#dexnet_21\n.. _Dex-Net 3.0: https://berkeleyautomation.github.io/dex-net/#dexnet_3\n.. _Dex-Net 4.0: https://berkeleyautomation.github.io/dex-net/#dexnet_4\n.. _FC-GQ-CNN: https://berkeleyautomation.github.io/fcgqcnn\n\nOur longer-term goal is to encourage development of robust GQ-CNNs that can be used to plan grasps on different hardware setups with different robots and cameras.\n\nDisclaimer\n----------\nGQ-CNN models are sensitive to the following parameters used during dataset generation:\n #. The robot gripper\n #. The depth camera\n #. The distance between the camera and workspace.\n\nAs a result, we cannot guarantee performance of our pre-trained models on other physical setups. If you have a specific use-case in mind, please reach out to us. It might be possible to generate a custom dataset for you particular setup. We are actively researching how to generate more robust datasets that can generalize across robots, cameras, and viewpoints.\n\nDevelopment\n-----------\nThe package is currently under active development. Please raise all bugs, feature requests, and other issues under the `Github Issues`_.\nFor other questions or concerns, please contact Jeff Mahler (jmahler@berkeley.edu) or Vishal Satish (vsatish@berkeley.edu) with the subject line starting with \"GQ-CNN Development\".\n\nAcademic Use\n------------\nIf you use the code, datasets, or models in a publication, please cite the appropriate paper:\n\n#. **Dex-Net 2.0** `(bibtex) `__\n#. **Dex-Net 2.1** `(bibtex) `__\n#. **Dex-Net 3.0** `(bibtex) `__\n#. **Dex-Net 4.0** `(bibtex) `__\n#. **FC-GQ-CNN** `(bibtex) `__\n\n.. _Grasp Quality Convolutional Neural Networks (GQ-CNNs): info/info.html\n\n.. _Dexterity Network (Dex-Net): https://berkeleyautomation.github.io/dex-net\n\n.. _Dex-Net: https://berkeleyautomation.github.io/dex-net\n\n.. _Berkeley AUTOLAB: http://autolab.berkeley.edu/\n\n.. _Dex-Net 2.0 paper: https://github.com/BerkeleyAutomation/dex-net/raw/gh-pages/docs/dexnet_rss2017_final.pdf\n\n.. _Github Issues: https://github.com/BerkeleyAutomation/gqcnn/issues\n\n.. toctree::\n :maxdepth: 2\n :caption: Background\n\n info/info.rst\n\n.. toctree::\n :maxdepth: 2\n :caption: Installation Guide\n\n install/install.rst\n\n.. toctree::\n :maxdepth: 2\n :caption: Getting Started\n\n tutorials/tutorial.rst\n\n.. toctree::\n :maxdepth: 2\n :caption: Replication\n\n replication/replication.rst\n\n.. toctree::\n :maxdepth: 2\n :caption: Benchmarks\n\n benchmarks/benchmarks.rst\n\n.. toctree::\n :maxdepth: 2\n :caption: API Documentation\n :glob:\n\n api/gqcnn.rst\n api/training.rst\n api/analysis.rst\n api/policies.rst\n\n.. toctree::\n :maxdepth: 2\n :caption: License\n\n license/license.rst\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "docs/source/info/info.rst", "content": "What are GQ-CNNs?\n-----------------\nGQ-CNNs are neural network architectures that take as input a depth image and grasp, and output the predicted probability that the grasp will successfully hold the object while lifting, transporting, and shaking the object.\n\n.. figure:: ../images/gqcnn.png\n :width: 100%\n :align: center\n\n Original GQ-CNN architecture from `Dex-Net 2.0`_.\n\n.. figure:: ../images/fcgqcnn_arch_diagram.png\n :width: 100%\n :align: center\n\n Alternate faster GQ-CNN architecture from `FC-GQ-CNN`_.\n\n\nThe GQ-CNN weights are trained on datasets of synthetic point clouds, parallel jaw grasps, and grasp metrics generated from physics-based models with domain randomization for sim-to-real transfer. See the ongoing `Dexterity Network (Dex-Net)`_ project for more information.\n\n.. _Dexterity Network (Dex-Net): https://berkeleyautomation.github.io/dex-net\n.. _Dex-Net 2.0: https://berkeleyautomation.github.io/dex-net/#dexnet_2\n.. _FC-GQ-CNN: https://berkeleyautomation.github.io/fcgqcnn\n" }, { "path": "docs/source/install/install.rst", "content": "Prerequisites\n~~~~~~~~~~~~~\n\nPython\n\"\"\"\"\"\"\n\nThe `gqcnn` package has only been tested with `Python 3.5`, `Python 3.6`, and `Python 3.7`.\n\nUbuntu\n\"\"\"\"\"\"\n\nThe `gqcnn` package has only been tested with `Ubuntu 12.04`, `Ubuntu 14.04` and `Ubuntu 16.04`.\n\nVirtualenv\n\"\"\"\"\"\"\"\"\"\"\n\nWe highly recommend using a Python environment management system, in particular `Virtualenv`, with the Pip and ROS installations. **Note: Several users have encountered problems with dependencies when using Conda.**\n\nPip Installation\n~~~~~~~~~~~~~~~~\n\nThe pip installation is intended for users who are **only interested in 1) Training GQ-CNNs or 2) Grasp planning on saved RGBD images**, not\ninterfacing with a physical robot.\nIf you have intentions of using GQ-CNNs for grasp planning on a physical robot, we suggest you `install as a ROS package`_.\n\n.. _install as a ROS package: https://berkeleyautomation.github.io/gqcnn/install/install.html#ros-installation\n\n1. Clone the repository\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\nClone or download the `project`_ from Github. ::\n\n $ git clone https://github.com/BerkeleyAutomation/gqcnn.git\n\n.. _project: https://github.com/BerkeleyAutomation/gqcnn\n\n2. Run pip installation\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\nChange directories into the `gqcnn` repository and run the pip installation. ::\n\n $ pip install .\n\nThis will install `gqcnn` in your current virtual environment.\n\n.. _ros-install:\n\nROS Installation\n~~~~~~~~~~~~~~~~\n\nInstallation as a ROS package is intended for users who wish to use GQ-CNNs to plan grasps on a physical robot.\n\n1. Clone the repository\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\nClone or download the `project`_ from Github. ::\n\n $ cd /src\n $ git clone https://github.com/BerkeleyAutomation/gqcnn.git\n\n2. Build the catkin package\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\nBuild the catkin package. ::\n\n $ cd \n $ catkin_make\n\nThen re-source `devel/setup.bash` for the package to be available through Python.\n\nDocker Installation\n~~~~~~~~~~~~~~~~~~~\n\nWe currently do not provide pre-built Docker images, but you can build them yourself. This will require you to have installed `Docker`_ or `Nvidia-Docker`_ if you plan on using GPUs. Note that our provided build for GPUs uses CUDA 10.0 and cuDNN 7.0, so make sure that this is compatible with your GPU hardware. If you wish to use a different CUDA/cuDNN version, change the base image in `docker/gpu/Dockerfile` to the desired `CUDA/cuDNN image distribution`_. **Note that other images have not yet been tested.**\n\n.. _Docker: https://www.docker.com/\n.. _Nvidia-Docker: https://github.com/NVIDIA/nvidia-docker\n.. _CUDA/cuDNN image distribution: https://hub.docker.com/r/nvidia/cuda/\n\n1. Clone the repository\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\nClone or download the `project`_ from Github. ::\n\n $ git clone https://github.com/BerkeleyAutomation/gqcnn.git\n\n.. _project: https://github.com/BerkeleyAutomation/gqcnn\n\n2. Build Docker images\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\nChange directories into the `gqcnn` repository and run the build script. ::\n\n $ ./scripts/docker/build-docker.sh\n\nThis will build the images `gqcnn/cpu` and `gqcnn/gpu`.\n\n3. Run Docker image\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\nTo run `gqcnn/cpu`: ::\n\n $ docker run --rm -it gqcnn/cpu\n\nTo run `gqcnn/gpu`: ::\n \n $ nvidia-docker run --rm -it gqcnn/gpu\n\nNote the use of `nvidia-docker` in the latter to enable the Nvidia runtime.\n\nYou will then see an interactive shell like this: ::\n\n $ root@a96488604093:~/Workspace/gqcnn#\n\nNow you can proceed to run the examples and tutorial!\n\n" }, { "path": "docs/source/license/license.rst", "content": "License\n~~~~~~~\nThe `gqcnn` library, pre-trained models, and raw datasets are licensed under The Regents of the University of California (Regents). Copyright ©2019. All Rights Reserved.\n\nPermission to use, copy, modify, and distribute this software and its documentation for educational,\nresearch, and not-for-profit purposes, without fee and without a signed licensing agreement, is\nhereby granted, provided that the above copyright notice, this paragraph and the following two\nparagraphs appear in all copies, modifications, and distributions. Contact The Office of Technology\nLicensing, UC Berkeley, 2150 Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643- \n7201, otl@berkeley.edu, http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nThe raw datasets are generated from 3D object models from `3DNet`_ and `the KIT Object Database`_ that may be subject to copyright.\n\n.. _3DNet: https://repo.acin.tuwien.ac.at/tmp/permanent/3d-net.org/\n.. _the KIT Object Database: https://h2t-projects.webarchiv.kit.edu/Projects/ObjectModelsWebUI/\n" }, { "path": "docs/source/replication/replication.rst", "content": "Replicating Results\n~~~~~~~~~~~~~~~~~~~\nNumerous publications in the larger `Dex-Net`_ project utilize GQ-CNNs, particularly `Dex-Net 2.0`_, `Dex-Net 2.1`_, `Dex-Net 3.0`_, `Dex-Net 4.0`_, and `FC-GQ-CNN`_. One of the goals of the `gqcnn` library is to provide the necessary code and instructions for replicating the results of these papers. Thus, we have provided a number of replication scripts under the `scripts/` directory.\n\nThere are two ways to replicate results:\n\n#. **Use a pre-trained model:** Download a pre-trained GQ-CNN model and run an example policy.\n#. **Train from scratch:** Download the raw dataset, train a GQ-CNN model, and run an example policy using the model you just trained.\n\n**We highly encourage method 1.** Note that method 2 is computationally expensive as training takes roughly 24 hours on a Nvidia Titan Xp GPU. Furthermore, the raw datasets are fairly large in size. \n\nPlease keep in mind that GQ-CNN models are sensitive to the following parameters used during dataset generation:\n #. The robot gripper\n #. The depth camera\n #. The distance between the camera and workspace.\n\nAs a result, we cannot guarantee performance of our pre-trained models on other physical setups.\n\nFor more information about the pre-trained models and sample inputs for the example policy, see :ref:`pre-trained-models` and :ref:`sample-inputs`.\n\nUsing a Pre-trained Model\n=========================\nFirst download the pre-trained models. ::\n\n $ ./scripts/downloads/models/download_models.sh\n\nDex-Net 2.0\n\"\"\"\"\"\"\"\"\"\"\"\nEvaluate the pre-trained GQ-CNN model. ::\n\n $ ./scripts/policies/run_all_dex-net_2.0_examples.sh\n\nDex-Net 2.1\n\"\"\"\"\"\"\"\"\"\"\"\nEvaluate the pre-trained GQ-CNN model. ::\n\n $ ./scripts/policies/run_all_dex-net_2.1_examples.sh\n\nDex-Net 3.0\n\"\"\"\"\"\"\"\"\"\"\"\nEvaluate the pre-trained GQ-CNN model. ::\n\n $ ./scripts/policies/run_all_dex-net_3.0_examples.sh\n\nDex-Net 4.0\n\"\"\"\"\"\"\"\"\"\"\"\nTo evaluate the pre-trained **parallel jaw** GQ-CNN model. ::\n\n $ ./scripts/policies/run_all_dex-net_4.0_pj_examples.sh\n\nTo evaluate the pre-trained **suction** GQ-CNN model. ::\n\n $ ./scripts/policies/run_all_dex-net_4.0_suction_examples.sh\n\nFC-GQ-CNN\n\"\"\"\"\"\"\"\"\"\"\"\nTo evaluate the pre-trained **parallel jaw** FC-GQ-CNN model. ::\n\n $ ./scripts/policies/run_all_dex-net_4.0_fc_pj_examples.sh\n\nTo evaluate the pre-trained **suction** FC-GQ-CNN model. ::\n\n $ ./scripts/policies/run_all_dex-net_4.0_fc_suction_examples.sh\n\n\n\nTraining from Scratch\n=====================\n\nDex-Net 2.0\n\"\"\"\"\"\"\"\"\"\"\"\nFirst download the appropriate dataset. ::\n\n $ ./scripts/downloads/datasets/download_dex-net_2.0.sh\n\nThen train a GQ-CNN from scratch. ::\n\n $ ./scripts/training/train_dex-net_2.0.sh\n\nFinally, evaluate the trained GQ-CNN. :: \n\n $ ./scripts/policies/run_all_dex-net_2.0_examples.sh \n\nDex-Net 2.1\n\"\"\"\"\"\"\"\"\"\"\"\nFirst download the appropriate dataset. ::\n\n $ ./scripts/downloads/datasets/download_dex-net_2.1.sh\n\nThen train a GQ-CNN from scratch. ::\n\n $ ./scripts/training/train_dex-net_2.1.sh\n\nFinally, evaluate the trained GQ-CNN. :: \n\n $ ./scripts/policies/run_all_dex-net_2.1_examples.sh \n\nDex-Net 3.0\n\"\"\"\"\"\"\"\"\"\"\"\nFirst download the appropriate dataset. ::\n\n $ ./scripts/downloads/datasets/download_dex-net_3.0.sh\n\nThen train a GQ-CNN from scratch. ::\n\n $ ./scripts/training/train_dex-net_3.0.sh\n\nFinally, evaluate the trained GQ-CNN. :: \n\n $ ./scripts/policies/run_all_dex-net_3.0_examples.sh \n\nDex-Net 4.0\n\"\"\"\"\"\"\"\"\"\"\"\nTo replicate the `Dex-Net 4.0`_ **parallel jaw** results, first download the appropriate dataset. ::\n\n $ ./scripts/downloads/datasets/download_dex-net_4.0_pj.sh\n\nThen train a GQ-CNN from scratch. ::\n\n $ ./scripts/training/train_dex-net_4.0_pj.sh\n\nFinally, evaluate the trained GQ-CNN. :: \n\n $ ./scripts/policies/run_all_dex-net_4.0_pj_examples.sh\n\nTo replicate the `Dex-Net 4.0`_ **suction** results, first download the appropriate dataset. ::\n\n $ ./scripts/downloads/datasets/download_dex-net_4.0_suction.sh\n\nThen train a GQ-CNN from scratch. ::\n\n $ ./scripts/training/train_dex-net_4.0_suction.sh\n\nFinally, evaluate the trained GQ-CNN. :: \n\n $ ./scripts/policies/run_all_dex-net_4.0_suction_examples.sh\n\nFC-GQ-CNN\n\"\"\"\"\"\"\"\"\"\"\"\nTo replicate the `FC-GQ-CNN`_ **parallel jaw** results, first download the appropriate dataset. ::\n\n $ ./scripts/downloads/datasets/download_dex-net_4.0_fc_pj.sh\n\nThen train a FC-GQ-CNN from scratch. ::\n\n $ ./scripts/training/train_dex-net_4.0_fc_pj.sh\n\nFinally, evaluate the trained FC-GQ-CNN. :: \n\n $ ./scripts/policies/run_all_dex-net_4.0_fc_pj_examples.sh\n\nTo replicate the `FC-GQ-CNN`_ **suction** results, first download the appropriate dataset. ::\n\n $ ./scripts/downloads/datasets/download_dex-net_4.0_fc_suction.sh\n\nThen train a FC-GQ-CNN from scratch. ::\n\n $ ./scripts/training/train_dex-net_4.0_fc_suction.sh\n\nFinally, evaluate the trained FC-GQ-CNN. :: \n\n $ ./scripts/policies/run_all_dex-net_4.0_fc_suction_examples.sh\n\n.. _Dex-Net: https://berkeleyautomation.github.io/dex-net/\n.. _Dex-Net 2.0: https://berkeleyautomation.github.io/dex-net/#dexnet_2\n.. _Dex-Net 2.1: https://berkeleyautomation.github.io/dex-net/#dexnet_21\n.. _Dex-Net 3.0: https://berkeleyautomation.github.io/dex-net/#dexnet_3\n.. _Dex-Net 4.0: https://berkeleyautomation.github.io/dex-net/#dexnet_4\n.. _FC-GQ-CNN: https://berkeleyautomation.github.io/fcgqcnn \n.. _gqcnn: https://github.com/BerkeleyAutomation/gqcnn\n\n" }, { "path": "docs/source/tutorials/analysis.rst", "content": "Analysis\n~~~~~~~~\nIt is helpful to check the training and validation loss and classification errors to ensure that the network has trained successfully. To analyze the performance of a trained GQ-CNN, run: ::\n\n $ python tools/analyze_gqcnn_performance.py \n\nThe args are:\n\n#. **model_name**: Name of a trained model.\n\nThe script will store a detailed analysis in `analysis//`.\n\nTo analyze the networks we just trained, run: ::\n\n $ python tools/analyze_gqcnn_performance.py gqcnn_example_pj\n $ python tools/analyze_gqcnn_performance.py gqcnn_example_suction\n\nBelow is the expected output for the **parallel jaw** network. Please keep in mind that the exact performance values may change due to randomization in the training dataset and random weight initialization: ::\n\n $ GQCNNAnalyzer INFO TRAIN\n $ GQCNNAnalyzer INFO Original error: 36.812\n $ GQCNNAnalyzer INFO Final error: 6.061\n $ GQCNNAnalyzer INFO Orig loss: 0.763\n $ GQCNNAnalyzer INFO Final loss: 0.248\n $ GQCNNAnalyzer INFO VAL\n $ GQCNNAnalyzer INFO Original error: 32.212\n $ GQCNNAnalyzer INFO Final error: 7.509\n $ GQCNNAnalyzer INFO Normalized error: 0.233\n\nA set of plots will be saved to `analysis/gqcnn_example_pj/`. The plots `training_error_rates.png` and `precision_recall.png` should look like the following:\n\n.. image:: ../images/plots/pj_error_rate.png\n :width: 49 %\n\n.. image:: ../images/plots/pj_roc.png\n :width: 49 %\n\nHere is the expected output for the **suction** network: ::\n\n $ GQCNNAnalyzer INFO TRAIN\n $ GQCNNAnalyzer INFO Original error: 17.844\n $ GQCNNAnalyzer INFO Final error: 6.417\n $ GQCNNAnalyzer INFO Orig loss: 0.476\n $ GQCNNAnalyzer INFO Final loss: 0.189\n $ GQCNNAnalyzer INFO VAL\n $ GQCNNAnalyzer INFO Original error: 18.036\n $ GQCNNAnalyzer INFO Final error: 6.907\n $ GQCNNAnalyzer INFO Normalized error: 0.383\n\nA set of plots will be saved to `analysis/gqcnn_example_suction/`. The plots `training_error_rates.png` and `precision_recall.png` should look like the following:\n\n.. image:: ../images/plots/suction_error_rate.png\n :width: 49 %\n\n.. image:: ../images/plots/suction_roc.png\n :width: 49 %\n\n" }, { "path": "docs/source/tutorials/planning.rst", "content": "Grasp Planning\n~~~~~~~~~~~~~~\nGrasp planning involves searching for the grasp with the highest predicted probability of success given a point cloud.\nIn the `gqcnn` package this is implemented as policies that map an RGBD image to a 6-DOF grasping pose by maximizing the output of a GQ-CNN. The maximization can be implemented with iterative methods such as the `Cross Entropy Method (CEM)`_, which is used in `Dex-Net 2.0`_, `Dex-Net 2.1`_, `Dex-Net 3.0`_, `Dex-Net 4.0`_, or much faster `fully convolutional networks`, which are used in the `FC-GQ-CNN`_. \n\n.. _Cross Entropy Method (CEM): https://en.wikipedia.org/wiki/Cross-entropy_method\n.. _Dex-Net 2.0: https://berkeleyautomation.github.io/dex-net/#dexnet_2\n.. _Dex-Net 2.1: https://berkeleyautomation.github.io/dex-net/#dexnet_21\n.. _Dex-Net 3.0: https://berkeleyautomation.github.io/dex-net/#dexnet_3\n.. _Dex-Net 4.0: https://berkeleyautomation.github.io/dex-net/#dexnet_4\n.. _FC-GQ-CNN: https://berkeleyautomation.github.io/fcgqcnn\n\nWe provide example policies in `examples/`. In particular, we provide both an example Python policy and an example ROS policy. **Note that the ROS policy requires the ROS gqcnn installation**, which can be found :ref:`here `. We highly recommend using the Python policy unless you need to plan grasps on a physical robot using ROS.\n\n.. _sample-inputs:\n\nSample Inputs\n-------------\nSample inputs from our experimental setup are provided with the repo:\n\n#. **data/examples/clutter/phoxi/dex-net_4.0**: Set of example images from a PhotoNeo PhoXi S containing objects used in `Dex-Net 4.0`_ experiments arranged in heaps. \n#. **data/examples/clutter/phoxi/fcgqcnn**: Set of example images from a PhotoNeo PhoXi S containing objects in `FC-GQ-CNN`_ experiments arranged in heaps.\n#. **data/examples/single_object/primesense/**: Set of example images from a Primesense Carmine containing objects used in `Dex-Net 2.0`_ experiments in singulation. \n#. **data/examples/clutter/primesense/**: Set of example images from a Primesense Carmine containing objects used in `Dex-Net 2.1`_ experiments arranged in heaps.\n\n**\\*\\*Note that when trying these sample inputs, you must make sure that the GQ-CNN model you are using was trained for the corresponding camera and input type (singulation/clutter). See the following section for more details.\\*\\***\n\n.. _pre-trained-models:\n\nPre-trained Models\n------------------\nPre-trained parallel jaw and suction models for `Dex-Net 4.0`_ are automatically downloaded with the `gqcnn` package installation. If you do wish to try out models for older results (or for our experimental `FC-GQ-CNN`_), all pre-trained models can be downloaded with: ::\n\n $ ./scripts/downloads/models/download_models.sh\n\nThe models are: \n\n#. **GQCNN-2.0**: For `Dex-Net 2.0`_, trained on images of objects in singulation with parameters for a Primesense Carmine.\n#. **GQCNN-2.1**: For `Dex-Net 2.1`_, a `Dex-Net 2.0`_ model fine-tuned on images of objects in clutter with parameters for a Primesense Carmine.\n#. **GQCNN-3.0**: For `Dex-Net 3.0`_, trained on images of objects in clutter with parameters for a Primesense Carmine.\n#. **GQCNN-4.0-PJ**: For `Dex-Net 4.0`_, trained on images of objects in clutter with parameters for a PhotoNeo PhoXi S.\n#. **GQCNN-4.0-SUCTION**: For `Dex-Net 4.0`_, trained on images of objects in clutter with parameters for a PhotoNeo PhoXi S.\n#. **FC-GQCNN-4.0-PJ**: For `FC-GQ-CNN`_, trained on images of objects in clutter with parameters for a PhotoNeo PhoXi S.\n#. **FC-GQCNN-4.0-SUCTION**: For `FC-GQ-CNN`_, trained on images of objects in clutter with parameters for a PhotoNeo PhoXi S. \n\n**\\*\\*Note that GQ-CNN models are sensitive to the parameters used during dataset generation, specifically 1) Gripper geometry, an ABB YuMi Parallel Jaw Gripper for all our pre-trained models 2) Camera intrinsics, either a Primesense Carmine or PhotoNeo Phoxi S for all our pre-trained models (see above for which one) 3) Distance between camera and workspace during rendering, 50-70cm for all our pre-trained models. Thus we cannot guarantee performance of our pre-trained models on other physical setups. If you have a specific use-case in mind, please reach out to us.\\*\\*** We are actively researching how to generate more robust datasets that can generalize across robots, cameras, and viewpoints!\n\nPython Policy\n-------------\nThe example Python policy can be queried on saved images using: ::\n\n $ python examples/policy.py --depth_image --segmask --camera_intr \n\nThe args are:\n\n#. **model_name**: Name of the GQ-CNN model to use.\n#. **depth_image_filename**: Path to a depth image (float array in .npy format).\n#. **segmask_filename**: Path to an object segmentation mask (binary image in .png format). \n#. **camera_intr_filename**: Path to a camera intrinsics file (.intr file generated with `BerkeleyAutomation's`_ `autolab_core`_ package).\n\n\n.. _BerkeleyAutomation's: https://github.com/BerkeleyAutomation\n.. _autolab_core: https://github.com/BerkeleyAutomation/autolab_core\n\nTo evaluate the pre-trained `Dex-Net 4.0`_ **parallel jaw** network on sample images of objects in heaps run: ::\n\n $ python examples/policy.py GQCNN-4.0-PJ --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_0.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_0.png --camera_intr data/calib/phoxi/phoxi.intr\n\nTo evaluate the pre-trained `Dex-Net 4.0`_ **suction** network on sample images of objects in heaps run: ::\n\n $ python examples/policy.py GQCNN-4.0-SUCTION --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_0.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_0.png --camera_intr data/calib/phoxi/phoxi.intr\n\n.. _ros-policy:\n\nROS Policy\n----------\nFirst start the grasp planning service: ::\n\n $ roslaunch gqcnn grasp_planning_service.launch model_name:=\n\nThe args are:\n\n#. **model_name**: Name of the GQ-CNN model to use. Default is **GQCNN-4.0-PJ**.\n#. **model_dir**: Path to the directory where the GQ-CNN models are located. Default is **models/**. If you are using the provided **download_models.sh** script, you shouldn't have to modify this.\n\nTo start the grasp planning service with the pre-trained `Dex-Net 4.0`_ **parallel jaw** network run: ::\n\n $ roslaunch gqcnn grasp_planning_service.launch model_name:=GQCNN-4.0-PJ\n\nTo start the grasp planning service with the pre-trained `Dex-Net 4.0`_ **suction** network run: ::\n\n $ roslaunch gqcnn grasp_planning_service.launch model_name:=GQCNN-4.0-SUCTION\n\nThe example ROS policy can then be queried on saved images using: ::\n\n $ python examples/policy_ros.py --depth_image --segmask --camera_intr \n\nThe args are:\n\n#. **depth_image_filename**: Path to a depth image (float array in .npy format).\n#. **segmask_filename**: Path to an object segmentation mask (binary image in .png format).\n#. **camera_intr_filename**: Path to a camera intrinsics file (.intr file generated with `BerkeleyAutomation's`_ `autolab_core`_ package).\n\nTo query the policy on sample images of objects in heaps run: ::\n\n $ python examples/policy_ros.py --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_0.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_0.png --camera_intr data/calib/phoxi/phoxi.intr\n\nUsage on a Physical Robot with ROS\n----------------------------------\nTo run the GQ-CNN on a physical robot with ROS, you will want to implement your own ROS node to query the grasp planning service similar to what `examples/policy_ros.py` does. If you are interested in replicating this functionality on your own robot, please contact Jeff Mahler (jmahler@berkeley.edu) with the subject line: \"Interested in GQ-CNN ROS Service\".\n\nFC-GQ-CNN Policy\n----------------\nOur most recent research result, the `FC-GQ-CNN`_, combines novel fully convolutional network architectures with our prior work on GQ-CNNs to increase policy rate and reliability. Instead of relying on the `Cross Entropy Method (CEM)`_ to iteratively search over the policy action space for the best grasp, the FC-GQ-CNN instead densely and efficiently evaluates the entire action space in parallel. It is thus able to consider 5000x more grasps in 0.625s, resulting in a MPPH (Mean Picks Per Hour) of 296, compared to the prior 250 MPPH of `Dex-Net 4.0`_.\n\n.. figure:: ../images/fcgqcnn_arch_diagram.png\n :width: 100 % \n :align: center\n\n FC-GQ-CNN architecture.\n\nYou can download the pre-trained `FC-GQ-CNN`_ parallel jaw and suction models along with the other pre-trained models: ::\n \n $ ./scripts/downloads/models/download_models.sh\n\nThen run the Python policy with the `\\\\--fully_conv` flag.\n\nTo evaluate the pre-trained `FC-GQ-CNN`_ **parallel jaw** network on sample images of objects in heaps run: ::\n\n $ python examples/policy.py FC-GQCNN-4.0-PJ --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_0.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_0.png --camera_intr data/calib/phoxi/phoxi.intr\n\nTo evaluate the pre-trained `FC-GQ-CNN`_ **suction** network on sample images of objects in heaps run: ::\n\n $ python examples/policy.py FC-GQCNN-4.0-SUCTION --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_0.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_0.png --camera_intr data/calib/phoxi/phoxi.intr\n\nWith ROS\n^^^^^^^^\n\nReview the section on using the normal ROS policy first, which can be found :ref:`here `.\n\nAdd the additional arg **fully_conv:=True** when launching the grasp planning service and provide the corresponding network (**FC-GQCNN-4.0-PJ** for **parallel jaw** and **FC-GQCNN-4.0-SUCTION** for **suction**).\n\nIf you wish to test on inputs other than those provided in `data/examples/clutter/phoxi/fcgqcnn/`, you will need to edit the input height and width configuration in the appropriate `cfg/examples/` under `[\"policy\"][\"metric\"][\"fully_conv_gqcnn_config\"]`.\n" }, { "path": "docs/source/tutorials/training.rst", "content": "Training\n~~~~~~~~\nThe `gqcnn` package can be used to train a `Dex-Net 4.0`_ GQ-CNN model on a custom offline `Dex-Net`_ dataset. Because training from scratch can be time-consuming, the most efficient way to train a new network is to fine-tune the weights of a pre-trained `Dex-Net 4.0`_ GQ-CNN model, which has already been trained on millions of images.\n\n.. _Dex-Net 4.0: https://berkeleyautomation.github.io/dex-net/#dexnet_4\n.. _Dex-Net: https://berkeleyautomation.github.io/dex-net/\n\nTo fine-tune a GQ-CNN run: ::\n\n $ python tools/finetune.py --config_filename --name \n\nThe args are:\n\n#. **training_dataset_path**: Path to the training dataset.\n#. **pretrained_network_name**: Name of pre-trained GQ-CNN.\n#. **config_filename**: Name of the config file to use.\n#. **model_name**: Name for the model.\n\nTo train a GQ-CNN for a **parallel jaw** gripper on a sample dataset, run the fine-tuning script: ::\n\n $ python tools/finetune.py data/training/example_pj/ GQCNN-4.0-PJ --config_filename cfg/finetune_example_pj.yaml --name gqcnn_example_pj\n\nTo train a GQ-CNN for a **suction** gripper run: ::\n\n $ python tools/finetune.py data/training/example_suction/ GQCNN-4.0-SUCTION --config_filename cfg/finetune_example_suction.yaml --name gqcnn_example_suction\n\nVisualizing Training\n--------------------\nThe `gqcnn` model contains support for visualizing training progress through Tensorboard. Tensorboard is automatically launched when the training script is run and can be accessed by navigating to **localhost:6006** in a web browser. There you will find something like the following:\n\n.. image:: ../images/tensorboard.png\n :width: 100 % \n\nWhich displays useful training statistics such as validation error, minibatch loss, and learning rate.\n\nThe Tensorflow summaries are stored in `models//tensorboard_summaries/`.\n\n" }, { "path": "docs/source/tutorials/tutorial.rst", "content": "Overview\n~~~~~~~~\nThere are two main use cases of the `gqcnn` package:\n\n#. :ref:`training` a `Dex-Net 4.0`_ GQ-CNN model on an offline `Dex-Net`_ dataset of point clouds, grasps, and grasp success metrics, and then grasp planning on RGBD images.\n#. :ref:`grasp planning` on RGBD images using a pre-trained `Dex-Net 4.0`_ GQ-CNN model.\n\n.. _Dex-Net 4.0: https://berkeleyautomation.github.io/dex-net/#dexnet_4\n.. _Dex-Net: https://berkeleyautomation.github.io/dex-net/\n\nClick on the links or scroll down to get started!\n\nPrerequisites\n-------------\nBefore running the tutorials please download the example models and datasets: ::\n\n $ cd /path/to/your/gqcnn\n $ ./scripts/downloads/download_example_data.sh\n $ ./scripts/downloads/models/download_models.sh\n\n\nRunning Python Scripts\n----------------------\nAll `gqcnn` Python scripts are designed to be run from the top-level directory of your `gqcnn` repo by default. This is because every script takes in a YAML file specifying parameters for the script, and this YAML file is stored relative to the repository root directory.\n\nWe recommend that you run all scripts using this paradigm: ::\n\n cd /path/to/your/gqcnn\n python /path/to/script.py\n\n.. _training:\n.. include:: training.rst\n\n.. _analysis:\n.. include:: analysis.rst\n\n.. _grasp planning:\n.. include:: planning.rst\n\n" }, { "path": "examples/antipodal_grasp_sampling.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nDemonstrates image-based antipodal grasp candidate sampling, which is used in\nthe Cross Entropy Method (CEM)-based GQ-CNN grasping policy. Samples images\nfrom a BerkeleyAutomation/perception `RgbdSensor`.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nimport argparse\nimport os\n\nfrom autolab_core import RigidTransform, YamlConfig, Logger, RgbdImage\nfrom perception import RgbdSensorFactory\nfrom visualization import Visualizer2D as vis\n\nfrom gqcnn.grasping import AntipodalDepthImageGraspSampler\n\n# Set up logger.\nlogger = Logger.get_logger(\"examples/antipodal_grasp_sampling.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(description=(\n \"Sample antipodal grasps on a depth image from an RgbdSensor\"))\n parser.add_argument(\"--config_filename\",\n type=str,\n default=\"cfg/examples/antipodal_grasp_sampling.yaml\",\n help=\"path to configuration file to use\")\n args = parser.parse_args()\n config_filename = args.config_filename\n\n # Read config.\n config = YamlConfig(config_filename)\n sensor_type = config[\"sensor\"][\"type\"]\n sensor_frame = config[\"sensor\"][\"frame\"]\n num_grasp_samples = config[\"num_grasp_samples\"]\n gripper_width = config[\"gripper_width\"]\n inpaint_rescale_factor = config[\"inpaint_rescale_factor\"]\n visualize_sampling = config[\"visualize_sampling\"]\n sample_config = config[\"sampling\"]\n\n # Read camera calib.\n tf_filename = \"%s_to_world.tf\" % (sensor_frame)\n T_camera_world = RigidTransform.load(\n os.path.join(config[\"calib_dir\"], sensor_frame, tf_filename))\n\n # Setup sensor.\n sensor = RgbdSensorFactory.sensor(sensor_type, config[\"sensor\"])\n sensor.start()\n camera_intr = sensor.ir_intrinsics\n\n # Read images.\n color_im, depth_im, _ = sensor.frames()\n color_im = color_im.inpaint(rescale_factor=inpaint_rescale_factor)\n depth_im = depth_im.inpaint(rescale_factor=inpaint_rescale_factor)\n rgbd_im = RgbdImage.from_color_and_depth(color_im, depth_im)\n\n # Sample grasps.\n grasp_sampler = AntipodalDepthImageGraspSampler(sample_config,\n gripper_width)\n grasps = grasp_sampler.sample(rgbd_im,\n camera_intr,\n num_grasp_samples,\n segmask=None,\n seed=100,\n visualize=visualize_sampling)\n\n # Visualize.\n vis.figure()\n vis.imshow(depth_im)\n for grasp in grasps:\n vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True)\n vis.title(\"Sampled grasps\")\n vis.show()\n" }, { "path": "examples/policy.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nDisplays robust grasps planned using a GQ-CNN grapsing policy on a set of saved\nRGB-D images. The default configuration for the standard GQ-CNN policy is\n`cfg/examples/cfg/examples/gqcnn_pj.yaml`. The default configuration for the\nFully-Convolutional GQ-CNN policy is `cfg/examples/fc_gqcnn_pj.yaml`.\n\nAuthor\n------\nJeff Mahler & Vishal Satish\n\"\"\"\nimport argparse\nimport json\nimport os\nimport time\n\nimport numpy as np\n\nfrom autolab_core import (YamlConfig, Logger, BinaryImage, CameraIntrinsics,\n ColorImage, DepthImage, RgbdImage)\nfrom visualization import Visualizer2D as vis\n\nfrom gqcnn.grasping import (RobustGraspingPolicy,\n CrossEntropyRobustGraspingPolicy, RgbdImageState,\n FullyConvolutionalGraspingPolicyParallelJaw,\n FullyConvolutionalGraspingPolicySuction)\nfrom gqcnn.utils import GripperMode\n\n# Set up logger.\nlogger = Logger.get_logger(\"examples/policy.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(\n description=\"Run a grasping policy on an example image\")\n parser.add_argument(\"model_name\",\n type=str,\n default=None,\n help=\"name of a trained model to run\")\n parser.add_argument(\n \"--depth_image\",\n type=str,\n default=None,\n help=\"path to a test depth image stored as a .npy file\")\n parser.add_argument(\"--segmask\",\n type=str,\n default=None,\n help=\"path to an optional segmask to use\")\n parser.add_argument(\"--camera_intr\",\n type=str,\n default=None,\n help=\"path to the camera intrinsics\")\n parser.add_argument(\"--model_dir\",\n type=str,\n default=None,\n help=\"path to the folder in which the model is stored\")\n parser.add_argument(\"--config_filename\",\n type=str,\n default=None,\n help=\"path to configuration file to use\")\n parser.add_argument(\n \"--fully_conv\",\n action=\"store_true\",\n help=(\"run Fully-Convolutional GQ-CNN policy instead of standard\"\n \" GQ-CNN policy\"))\n args = parser.parse_args()\n model_name = args.model_name\n depth_im_filename = args.depth_image\n segmask_filename = args.segmask\n camera_intr_filename = args.camera_intr\n model_dir = args.model_dir\n config_filename = args.config_filename\n fully_conv = args.fully_conv\n\n assert not (fully_conv and depth_im_filename is not None\n and segmask_filename is None\n ), \"Fully-Convolutional policy expects a segmask.\"\n\n if depth_im_filename is None:\n if fully_conv:\n depth_im_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/examples/clutter/primesense/depth_0.npy\")\n else:\n depth_im_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/examples/single_object/primesense/depth_0.npy\")\n if fully_conv and segmask_filename is None:\n segmask_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/examples/clutter/primesense/segmask_0.png\")\n if camera_intr_filename is None:\n camera_intr_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/calib/primesense/primesense.intr\")\n\n # Set model if provided.\n if model_dir is None:\n model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"../models\")\n model_path = os.path.join(model_dir, model_name)\n\n # Get configs.\n model_config = json.load(open(os.path.join(model_path, \"config.json\"),\n \"r\"))\n try:\n gqcnn_config = model_config[\"gqcnn\"]\n gripper_mode = gqcnn_config[\"gripper_mode\"]\n except KeyError:\n gqcnn_config = model_config[\"gqcnn_config\"]\n input_data_mode = gqcnn_config[\"input_data_mode\"]\n if input_data_mode == \"tf_image\":\n gripper_mode = GripperMode.LEGACY_PARALLEL_JAW\n elif input_data_mode == \"tf_image_suction\":\n gripper_mode = GripperMode.LEGACY_SUCTION\n elif input_data_mode == \"suction\":\n gripper_mode = GripperMode.SUCTION\n elif input_data_mode == \"multi_suction\":\n gripper_mode = GripperMode.MULTI_SUCTION\n elif input_data_mode == \"parallel_jaw\":\n gripper_mode = GripperMode.PARALLEL_JAW\n else:\n raise ValueError(\n \"Input data mode {} not supported!\".format(input_data_mode))\n\n # Set config.\n if config_filename is None:\n if (gripper_mode == GripperMode.LEGACY_PARALLEL_JAW\n or gripper_mode == GripperMode.PARALLEL_JAW):\n if fully_conv:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/fc_gqcnn_pj.yaml\")\n else:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/gqcnn_pj.yaml\")\n elif (gripper_mode == GripperMode.LEGACY_SUCTION\n or gripper_mode == GripperMode.SUCTION):\n if fully_conv:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/fc_gqcnn_suction.yaml\")\n else:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/gqcnn_suction.yaml\")\n\n # Read config.\n config = YamlConfig(config_filename)\n inpaint_rescale_factor = config[\"inpaint_rescale_factor\"]\n policy_config = config[\"policy\"]\n\n # Make relative paths absolute.\n if \"gqcnn_model\" in policy_config[\"metric\"]:\n policy_config[\"metric\"][\"gqcnn_model\"] = model_path\n if not os.path.isabs(policy_config[\"metric\"][\"gqcnn_model\"]):\n policy_config[\"metric\"][\"gqcnn_model\"] = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n policy_config[\"metric\"][\"gqcnn_model\"])\n\n # Setup sensor.\n camera_intr = CameraIntrinsics.load(camera_intr_filename)\n\n # Read images.\n depth_data = np.load(depth_im_filename)\n depth_im = DepthImage(depth_data, frame=camera_intr.frame)\n color_im = ColorImage(np.zeros([depth_im.height, depth_im.width,\n 3]).astype(np.uint8),\n frame=camera_intr.frame)\n\n # Optionally read a segmask.\n segmask = None\n if segmask_filename is not None:\n segmask = BinaryImage.open(segmask_filename)\n valid_px_mask = depth_im.invalid_pixel_mask().inverse()\n if segmask is None:\n segmask = valid_px_mask\n else:\n segmask = segmask.mask_binary(valid_px_mask)\n\n # Inpaint.\n depth_im = depth_im.inpaint(rescale_factor=inpaint_rescale_factor)\n\n if \"input_images\" in policy_config[\"vis\"] and policy_config[\"vis\"][\n \"input_images\"]:\n vis.figure(size=(10, 10))\n num_plot = 1\n if segmask is not None:\n num_plot = 2\n vis.subplot(1, num_plot, 1)\n vis.imshow(depth_im)\n if segmask is not None:\n vis.subplot(1, num_plot, 2)\n vis.imshow(segmask)\n vis.show()\n\n # Create state.\n rgbd_im = RgbdImage.from_color_and_depth(color_im, depth_im)\n state = RgbdImageState(rgbd_im, camera_intr, segmask=segmask)\n\n # Set input sizes for fully-convolutional policy.\n if fully_conv:\n policy_config[\"metric\"][\"fully_conv_gqcnn_config\"][\n \"im_height\"] = depth_im.shape[0]\n policy_config[\"metric\"][\"fully_conv_gqcnn_config\"][\n \"im_width\"] = depth_im.shape[1]\n\n # Init policy.\n if fully_conv:\n # TODO(vsatish): We should really be doing this in some factory policy.\n if policy_config[\"type\"] == \"fully_conv_suction\":\n policy = FullyConvolutionalGraspingPolicySuction(policy_config)\n elif policy_config[\"type\"] == \"fully_conv_pj\":\n policy = FullyConvolutionalGraspingPolicyParallelJaw(policy_config)\n else:\n raise ValueError(\n \"Invalid fully-convolutional policy type: {}\".format(\n policy_config[\"type\"]))\n else:\n policy_type = \"cem\"\n if \"type\" in policy_config:\n policy_type = policy_config[\"type\"]\n if policy_type == \"ranking\":\n policy = RobustGraspingPolicy(policy_config)\n elif policy_type == \"cem\":\n policy = CrossEntropyRobustGraspingPolicy(policy_config)\n else:\n raise ValueError(\"Invalid policy type: {}\".format(policy_type))\n\n # Query policy.\n policy_start = time.time()\n action = policy(state)\n logger.info(\"Planning took %.3f sec\" % (time.time() - policy_start))\n\n # Vis final grasp.\n if policy_config[\"vis\"][\"final_grasp\"]:\n vis.figure(size=(10, 10))\n vis.imshow(rgbd_im.depth,\n vmin=policy_config[\"vis\"][\"vmin\"],\n vmax=policy_config[\"vis\"][\"vmax\"])\n vis.grasp(action.grasp, scale=2.5, show_center=False, show_axis=True)\n vis.title(\"Planned grasp at depth {0:.3f}m with Q={1:.3f}\".format(\n action.grasp.depth, action.q_value))\n vis.show()\n" }, { "path": "examples/policy_ros.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nDisplays robust grasps planned using a GQ-CNN-based policy on a set of saved\nRGB-D images. The default configuration is cfg/examples/policy.yaml.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nimport argparse\nimport logging\nimport numpy as np\nimport os\nimport rosgraph.roslogging as rl\nimport rospy\nimport sys\n\nfrom cv_bridge import CvBridge, CvBridgeError\n\nfrom autolab_core import (Point, Logger, BinaryImage, CameraIntrinsics,\n ColorImage, DepthImage)\nfrom visualization import Visualizer2D as vis\n\nfrom gqcnn.grasping import Grasp2D, SuctionPoint2D, GraspAction\nfrom gqcnn.msg import GQCNNGrasp\nfrom gqcnn.srv import GQCNNGraspPlanner, GQCNNGraspPlannerSegmask\n\n# Set up logger.\nlogger = Logger.get_logger(\"examples/policy_ros.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(\n description=\"Run a grasping policy on an example image\")\n parser.add_argument(\n \"--depth_image\",\n type=str,\n default=None,\n help=\"path to a test depth image stored as a .npy file\")\n parser.add_argument(\"--segmask\",\n type=str,\n default=None,\n help=\"path to an optional segmask to use\")\n parser.add_argument(\"--camera_intr\",\n type=str,\n default=None,\n help=\"path to the camera intrinsics\")\n parser.add_argument(\"--gripper_width\",\n type=float,\n default=0.05,\n help=\"width of the gripper to plan for\")\n parser.add_argument(\"--namespace\",\n type=str,\n default=\"gqcnn\",\n help=\"namespace of the ROS grasp planning service\")\n parser.add_argument(\"--vis_grasp\",\n type=bool,\n default=True,\n help=\"whether or not to visualize the grasp\")\n args = parser.parse_args()\n depth_im_filename = args.depth_image\n segmask_filename = args.segmask\n camera_intr_filename = args.camera_intr\n gripper_width = args.gripper_width\n namespace = args.namespace\n vis_grasp = args.vis_grasp\n\n # Initialize the ROS node.\n rospy.init_node(\"grasp_planning_example\")\n logging.getLogger().addHandler(rl.RosStreamHandler())\n\n # Setup filenames.\n if depth_im_filename is None:\n depth_im_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/examples/single_object/primesense/depth_0.npy\")\n if camera_intr_filename is None:\n camera_intr_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/calib/primesense/primesense.intr\")\n\n # Wait for grasp planning service and create service proxy.\n rospy.wait_for_service(\"%s/grasp_planner\" % (namespace))\n rospy.wait_for_service(\"%s/grasp_planner_segmask\" % (namespace))\n plan_grasp = rospy.ServiceProxy(\"%s/grasp_planner\" % (namespace),\n GQCNNGraspPlanner)\n plan_grasp_segmask = rospy.ServiceProxy(\n \"%s/grasp_planner_segmask\" % (namespace), GQCNNGraspPlannerSegmask)\n cv_bridge = CvBridge()\n\n # Set up sensor.\n camera_intr = CameraIntrinsics.load(camera_intr_filename)\n\n # Read images.\n depth_im = DepthImage.open(depth_im_filename, frame=camera_intr.frame)\n color_im = ColorImage(np.zeros([depth_im.height, depth_im.width,\n 3]).astype(np.uint8),\n frame=camera_intr.frame)\n\n # Read segmask.\n if segmask_filename is not None:\n segmask = BinaryImage.open(segmask_filename, frame=camera_intr.frame)\n grasp_resp = plan_grasp_segmask(color_im.rosmsg, depth_im.rosmsg,\n camera_intr.rosmsg, segmask.rosmsg)\n else:\n grasp_resp = plan_grasp(color_im.rosmsg, depth_im.rosmsg,\n camera_intr.rosmsg)\n grasp = grasp_resp.grasp\n\n # Convert to a grasp action.\n grasp_type = grasp.grasp_type\n if grasp_type == GQCNNGrasp.PARALLEL_JAW:\n center = Point(np.array([grasp.center_px[0], grasp.center_px[1]]),\n frame=camera_intr.frame)\n grasp_2d = Grasp2D(center,\n grasp.angle,\n grasp.depth,\n width=gripper_width,\n camera_intr=camera_intr)\n elif grasp_type == GQCNNGrasp.SUCTION:\n center = Point(np.array([grasp.center_px[0], grasp.center_px[1]]),\n frame=camera_intr.frame)\n grasp_2d = SuctionPoint2D(center,\n np.array([0, 0, 1]),\n grasp.depth,\n camera_intr=camera_intr)\n else:\n raise ValueError(\"Grasp type %d not recognized!\" % (grasp_type))\n try:\n thumbnail = DepthImage(cv_bridge.imgmsg_to_cv2(\n grasp.thumbnail, desired_encoding=\"passthrough\"),\n frame=camera_intr.frame)\n except CvBridgeError as e:\n logger.error(e)\n logger.error(\"Failed to convert image\")\n sys.exit(1)\n action = GraspAction(grasp_2d, grasp.q_value, thumbnail)\n\n # Vis final grasp.\n if vis_grasp:\n vis.figure(size=(10, 10))\n vis.imshow(depth_im, vmin=0.6, vmax=0.9)\n vis.grasp(action.grasp, scale=2.5, show_center=False, show_axis=True)\n vis.title(\"Planned grasp on depth (Q=%.3f)\" % (action.q_value))\n vis.show()\n" }, { "path": "examples/policy_with_image_proc.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nDisplays robust grasps planned using a GQ-CNN-based policy on a set of saved\nRGB-D images. The default configuration is cfg/examples/policy.yaml.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nimport argparse\nimport os\nimport time\n\nimport numpy as np\nimport pcl\nimport skimage\n\nfrom autolab_core import (PointCloud, RigidTransform, YamlConfig,\n Logger, BinaryImage, CameraIntrinsics,\n ColorImage, DepthImage, RgbdImage,\n SegmentationImage)\nfrom visualization import Visualizer2D as vis\n\nfrom gqcnn import (RobustGraspingPolicy, CrossEntropyRobustGraspingPolicy,\n RgbdImageState)\n\nCLUSTER_TOL = 0.0015\nMIN_CLUSTER_SIZE = 100\nMAX_CLUSTER_SIZE = 1000000\n\n# Set up logger.\nlogger = Logger.get_logger(\"tools/policy_with_image_proc.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(\n description=\"Run a grasping policy on an example image\")\n parser.add_argument(\n \"--depth_image\",\n type=str,\n default=None,\n help=\"path to a test depth image stored as a .npy file\")\n parser.add_argument(\"--segmask\",\n type=str,\n default=None,\n help=\"path to an optional segmask to use\")\n parser.add_argument(\"--camera_intrinsics\",\n type=str,\n default=None,\n help=\"path to the camera intrinsics\")\n parser.add_argument(\"--camera_pose\",\n type=str,\n default=None,\n help=\"path to the camera pose\")\n parser.add_argument(\"--model_dir\",\n type=str,\n default=None,\n help=\"path to a trained model to run\")\n parser.add_argument(\"--config_filename\",\n type=str,\n default=None,\n help=\"path to configuration file to use\")\n args = parser.parse_args()\n depth_im_filename = args.depth_image\n segmask_filename = args.segmask\n camera_intr_filename = args.camera_intrinsics\n camera_pose_filename = args.camera_pose\n model_dir = args.model_dir\n config_filename = args.config_filename\n\n if depth_im_filename is None:\n depth_im_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/examples/single_object/depth_0.npy\")\n if camera_intr_filename is None:\n camera_intr_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/calib/primesense.intr\")\n if camera_pose_filename is None:\n camera_pose_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"data/calib/primesense.tf\")\n if config_filename is None:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/replication/dex-net_2.0.yaml\")\n\n # Read config.\n config = YamlConfig(config_filename)\n inpaint_rescale_factor = config[\"inpaint_rescale_factor\"]\n policy_config = config[\"policy\"]\n\n # Make relative paths absolute.\n if model_dir is not None:\n policy_config[\"metric\"][\"gqcnn_model\"] = model_dir\n if \"gqcnn_model\" in policy_config[\"metric\"] and not os.path.isabs(\n policy_config[\"metric\"][\"gqcnn_model\"]):\n policy_config[\"metric\"][\"gqcnn_model\"] = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n policy_config[\"metric\"][\"gqcnn_model\"])\n\n # Setup sensor.\n camera_intr = CameraIntrinsics.load(camera_intr_filename)\n T_camera_world = RigidTransform.load(camera_pose_filename)\n\n # Read images.\n depth_data = np.load(depth_im_filename)\n depth_data = depth_data.astype(np.float32) / 1000.0\n depth_im = DepthImage(depth_data, frame=camera_intr.frame)\n color_im = ColorImage(np.zeros([depth_im.height, depth_im.width,\n 3]).astype(np.uint8),\n frame=camera_intr.frame)\n\n # Optionally read a segmask.\n mask = np.zeros((camera_intr.height, camera_intr.width, 1), dtype=np.uint8)\n c = np.array([165, 460, 500, 135])\n r = np.array([165, 165, 370, 370])\n rr, cc = skimage.draw.polygon(r, c, shape=mask.shape)\n mask[rr, cc, 0] = 255\n segmask = BinaryImage(mask)\n if segmask_filename is not None:\n segmask = BinaryImage.open(segmask_filename)\n valid_px_mask = depth_im.invalid_pixel_mask().inverse()\n if segmask is None:\n segmask = valid_px_mask\n else:\n segmask = segmask.mask_binary(valid_px_mask)\n\n # Create new cloud.\n point_cloud = camera_intr.deproject(depth_im)\n point_cloud.remove_zero_points()\n pcl_cloud = pcl.PointCloud(point_cloud.data.T.astype(np.float32))\n tree = pcl_cloud.make_kdtree()\n\n # Find large clusters (likely to be real objects instead of noise).\n ec = pcl_cloud.make_EuclideanClusterExtraction()\n ec.set_ClusterTolerance(CLUSTER_TOL)\n ec.set_MinClusterSize(MIN_CLUSTER_SIZE)\n ec.set_MaxClusterSize(MAX_CLUSTER_SIZE)\n ec.set_SearchMethod(tree)\n cluster_indices = ec.Extract()\n num_clusters = len(cluster_indices)\n\n obj_segmask_data = np.zeros(depth_im.shape)\n\n # Read out all points in large clusters.\n cur_i = 0\n for j, indices in enumerate(cluster_indices):\n num_points = len(indices)\n points = np.zeros([3, num_points])\n\n for i, index in enumerate(indices):\n points[0, i] = pcl_cloud[index][0]\n points[1, i] = pcl_cloud[index][1]\n points[2, i] = pcl_cloud[index][2]\n\n segment = PointCloud(points, frame=camera_intr.frame)\n depth_segment = camera_intr.project_to_image(segment)\n obj_segmask_data[depth_segment.data > 0] = j + 1\n obj_segmask = SegmentationImage(obj_segmask_data.astype(np.uint8))\n obj_segmask = obj_segmask.mask_binary(segmask)\n\n # Inpaint.\n depth_im = depth_im.inpaint(rescale_factor=inpaint_rescale_factor)\n\n if \"input_images\" in policy_config[\"vis\"] and policy_config[\"vis\"][\n \"input_images\"]:\n vis.figure(size=(10, 10))\n num_plot = 3\n vis.subplot(1, num_plot, 1)\n vis.imshow(depth_im)\n vis.subplot(1, num_plot, 2)\n vis.imshow(segmask)\n vis.subplot(1, num_plot, 3)\n vis.imshow(obj_segmask)\n vis.show()\n\n from visualization import Visualizer3D as vis3d\n point_cloud = camera_intr.deproject(depth_im)\n vis3d.figure()\n vis3d.points(point_cloud,\n subsample=3,\n random=True,\n color=(0, 0, 1),\n scale=0.001)\n vis3d.pose(RigidTransform())\n vis3d.pose(T_camera_world.inverse())\n vis3d.show()\n\n # Create state.\n rgbd_im = RgbdImage.from_color_and_depth(color_im, depth_im)\n state = RgbdImageState(rgbd_im, camera_intr, segmask=segmask)\n\n # Init policy.\n policy_type = \"cem\"\n if \"type\" in policy_config:\n policy_type = policy_config[\"type\"]\n if policy_type == \"ranking\":\n policy = RobustGraspingPolicy(policy_config)\n else:\n policy = CrossEntropyRobustGraspingPolicy(policy_config)\n policy_start = time.time()\n action = policy(state)\n logger.info(\"Planning took %.3f sec\" % (time.time() - policy_start))\n\n # Vis final grasp.\n if policy_config[\"vis\"][\"final_grasp\"]:\n vis.figure(size=(10, 10))\n vis.imshow(rgbd_im.depth,\n vmin=policy_config[\"vis\"][\"vmin\"],\n vmax=policy_config[\"vis\"][\"vmax\"])\n vis.grasp(action.grasp, scale=2.5, show_center=False, show_axis=True)\n vis.title(\"Planned grasp on depth (Q=%.3f)\" % (action.q_value))\n vis.show()\n\n # Get grasp pose.\n T_grasp_camera = action.grasp.pose(\n grasp_approach_dir=-T_camera_world.inverse().z_axis)\n grasp_pose_msg = T_grasp_camera.pose_msg\n\n # TODO: Control to reach the grasp pose.\n" }, { "path": "gqcnn/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\"\"\"\nfrom .model import get_gqcnn_model, get_fc_gqcnn_model\nfrom .training import get_gqcnn_trainer\nfrom .grasping import (RobustGraspingPolicy, UniformRandomGraspingPolicy,\n CrossEntropyRobustGraspingPolicy, RgbdImageState,\n FullyConvolutionalGraspingPolicyParallelJaw,\n FullyConvolutionalGraspingPolicySuction)\nfrom .analysis import GQCNNAnalyzer\nfrom .search import GQCNNSearch\nfrom .utils import NoValidGraspsException, NoAntipodalPairsFoundException\n\n__all__ = [\n \"get_gqcnn_model\", \"get_fc_gqcnn_model\", \"get_gqcnn_trainer\",\n \"RobustGraspingPolicy\", \"UniformRandomGraspingPolicy\",\n \"CrossEntropyRobustGraspingPolicy\", \"RgbdImageState\",\n \"FullyConvolutionalGraspingPolicyParallelJaw\",\n \"FullyConvolutionalGraspingPolicySuction\", \"GQCNNAnalyzer\", \"GQCNNSearch\",\n \"NoValidGraspsException\", \"NoAntipodalPairsFoundException\"\n]\n" }, { "path": "gqcnn/analysis/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\"\"\"\nfrom .analyzer import GQCNNAnalyzer\n\n__all__ = [\"GQCNNAnalyzer\"]\n" }, { "path": "gqcnn/analysis/analyzer.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nClass for analyzing a GQ-CNN model for grasp quality prediction.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nimport json\nimport logging\nimport os\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nfrom autolab_core import (BinaryClassificationResult, TensorDataset,\n Logger, DepthImage)\nfrom autolab_core.constants import JSON_INDENT\nimport autolab_core.utils as utils\nfrom visualization import Visualizer2D as vis2d\n\nfrom ..model import get_gqcnn_model\nfrom ..grasping import Grasp2D, SuctionPoint2D\nfrom ..utils import (GripperMode, GeneralConstants, read_pose_data,\n GQCNNFilenames)\n\nWINDOW = 100 # For loss.\nMAX_LOSS = 5.0\n\n\nclass GQCNNAnalyzer(object):\n \"\"\"Analyzes a trained GQ-CNN model.\"\"\"\n\n def __init__(self, config, verbose=True, plot_backend=\"pdf\"):\n \"\"\"\n Parameters\n ----------\n config : dict\n Dictionary of analysis configuration parameters.\n verbose : bool\n Whether or not to log analysis output to stdout.\n plot_backend : str\n Matplotlib plotting backend to use, default is non-interactive\n \"pdf\" backend.\n \"\"\"\n self.cfg = config\n self.verbose = verbose\n\n # By default we want to use a non-interactive backend (ex. \"pdf)\"\n # because the `GQCNNAnalyzer` anyways only saves plots to disk, and\n # interactive backends sometimes cause issues with `localhost` when run\n # remotely through the Linux `screen` program.\n plt.switch_backend(plot_backend)\n self._parse_config()\n\n def _parse_config(self):\n \"\"\"Read params from the config file.\"\"\"\n # Plotting params.\n self.log_rate = self.cfg[\"log_rate\"]\n self.font_size = self.cfg[\"font_size\"]\n self.line_width = self.cfg[\"line_width\"]\n self.dpi = self.cfg[\"dpi\"]\n self.num_bins = self.cfg[\"num_bins\"]\n self.num_vis = self.cfg[\"num_vis\"]\n\n def analyze(self, model_dir, output_dir, dataset_config=None):\n \"\"\"Run analysis.\n\n Parameters\n ----------\n model_dir : str\n Path to the GQ-CNN model to analyze.\n output_dir : str\n Path to save the analysis.\n dataset_config : dict\n Dictionary to configure dataset used for training evaluation if\n different from one used during training.\n\n Returns\n -------\n :obj:`autolab_core.BinaryClassificationResult`\n Result of analysis on training data.\n :obj:`autolab_core.BinaryClassificationResult`\n Result of analysis on validation data.\n \"\"\"\n # Determine model output dir.\n model_name = \"\"\n model_root = model_dir\n while model_name == \"\" and model_root != \"\":\n model_root, model_name = os.path.split(model_root)\n\n model_output_dir = os.path.join(output_dir, model_name)\n if not os.path.exists(model_output_dir):\n os.mkdir(model_output_dir)\n\n # Set up logger.\n self.logger = Logger.get_logger(self.__class__.__name__,\n log_file=os.path.join(\n model_output_dir, \"analysis.log\"),\n silence=(not self.verbose),\n global_log_file=self.verbose)\n\n self.logger.info(\"Analyzing model %s\" % (model_name))\n self.logger.info(\"Saving output to %s\" % (output_dir))\n\n # Run predictions.\n train_result, val_result = self._run_prediction_single_model(\n model_dir, model_output_dir, dataset_config)\n\n # Finally plot curves.\n (init_train_error, final_train_error, init_train_loss,\n final_train_loss, init_val_error, final_val_error,\n norm_final_val_error) = self._plot(model_dir, model_output_dir,\n train_result, val_result)\n\n return (train_result, val_result, init_train_error, final_train_error,\n init_train_loss, final_train_loss, init_val_error,\n final_val_error, norm_final_val_error)\n\n def _plot_grasp(self,\n datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=None):\n \"\"\"Plots a single grasp represented as a datapoint.\"\"\"\n image = DepthImage(datapoint[image_field_name][:, :, 0])\n depth = datapoint[pose_field_name][2]\n width = 0\n grasps = []\n if gripper_mode == GripperMode.PARALLEL_JAW or \\\n gripper_mode == GripperMode.LEGACY_PARALLEL_JAW:\n if angular_preds is not None:\n num_bins = angular_preds.shape[0] / 2\n bin_width = GeneralConstants.PI / num_bins\n for i in range(num_bins):\n bin_cent_ang = i * bin_width + bin_width / 2\n grasps.append(\n Grasp2D(center=image.center,\n angle=GeneralConstants.PI / 2 - bin_cent_ang,\n depth=depth,\n width=0.0))\n grasps.append(\n Grasp2D(center=image.center,\n angle=datapoint[pose_field_name][3],\n depth=depth,\n width=0.0))\n else:\n grasps.append(\n Grasp2D(center=image.center,\n angle=0,\n depth=depth,\n width=0.0))\n width = datapoint[pose_field_name][-1]\n else:\n grasps.append(\n SuctionPoint2D(center=image.center,\n axis=[1, 0, 0],\n depth=depth))\n vis2d.imshow(image)\n for i, grasp in enumerate(grasps[:-1]):\n vis2d.grasp(grasp,\n width=width,\n color=plt.cm.RdYlGn(angular_preds[i * 2 + 1]))\n vis2d.grasp(grasps[-1], width=width, color=\"b\")\n\n def _run_prediction_single_model(self, model_dir, model_output_dir,\n dataset_config):\n \"\"\"Analyze the performance of a single model.\"\"\"\n # Read in model config.\n model_config_filename = os.path.join(model_dir,\n GQCNNFilenames.SAVED_CFG)\n with open(model_config_filename) as data_file:\n model_config = json.load(data_file)\n\n # Load model.\n self.logger.info(\"Loading model %s\" % (model_dir))\n log_file = None\n for handler in self.logger.handlers:\n if isinstance(handler, logging.FileHandler):\n log_file = handler.baseFilename\n gqcnn = get_gqcnn_model(verbose=self.verbose).load(\n model_dir, verbose=self.verbose, log_file=log_file)\n gqcnn.open_session()\n gripper_mode = gqcnn.gripper_mode\n angular_bins = gqcnn.angular_bins\n\n # Read params from the config.\n if dataset_config is None:\n dataset_dir = model_config[\"dataset_dir\"]\n split_name = model_config[\"split_name\"]\n image_field_name = model_config[\"image_field_name\"]\n pose_field_name = model_config[\"pose_field_name\"]\n metric_name = model_config[\"target_metric_name\"]\n metric_thresh = model_config[\"metric_thresh\"]\n else:\n dataset_dir = dataset_config[\"dataset_dir\"]\n split_name = dataset_config[\"split_name\"]\n image_field_name = dataset_config[\"image_field_name\"]\n pose_field_name = dataset_config[\"pose_field_name\"]\n metric_name = dataset_config[\"target_metric_name\"]\n metric_thresh = dataset_config[\"metric_thresh\"]\n gripper_mode = dataset_config[\"gripper_mode\"]\n\n self.logger.info(\"Loading dataset %s\" % (dataset_dir))\n dataset = TensorDataset.open(dataset_dir)\n train_indices, val_indices, _ = dataset.split(split_name)\n\n # Visualize conv filters.\n conv1_filters = gqcnn.filters\n num_filt = conv1_filters.shape[3]\n d = utils.sqrt_ceil(num_filt)\n vis2d.clf()\n for k in range(num_filt):\n filt = conv1_filters[:, :, 0, k]\n vis2d.subplot(d, d, k + 1)\n vis2d.imshow(DepthImage(filt))\n figname = os.path.join(model_output_dir, \"conv1_filters.pdf\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n # Aggregate training and validation true labels and predicted\n # probabilities.\n all_predictions = []\n if angular_bins > 0:\n all_predictions_raw = []\n all_labels = []\n for i in range(dataset.num_tensors):\n # Log progress.\n if i % self.log_rate == 0:\n self.logger.info(\"Predicting tensor %d of %d\" %\n (i + 1, dataset.num_tensors))\n\n # Read in data.\n image_arr = dataset.tensor(image_field_name, i).arr\n pose_arr = read_pose_data(\n dataset.tensor(pose_field_name, i).arr, gripper_mode)\n metric_arr = dataset.tensor(metric_name, i).arr\n label_arr = 1 * (metric_arr > metric_thresh)\n label_arr = label_arr.astype(np.uint8)\n if angular_bins > 0:\n # Form mask to extract predictions from ground-truth angular\n # bins.\n raw_poses = dataset.tensor(pose_field_name, i).arr\n angles = raw_poses[:, 3]\n neg_ind = np.where(angles < 0)\n # TODO(vsatish): These should use the max angle instead.\n angles = np.abs(angles) % GeneralConstants.PI\n angles[neg_ind] *= -1\n g_90 = np.where(angles > (GeneralConstants.PI / 2))\n l_neg_90 = np.where(angles < (-1 * (GeneralConstants.PI / 2)))\n angles[g_90] -= GeneralConstants.PI\n angles[l_neg_90] += GeneralConstants.PI\n # TODO(vsatish): Fix this along with the others.\n angles *= -1 # Hack to fix reverse angle convention.\n angles += (GeneralConstants.PI / 2)\n pred_mask = np.zeros((raw_poses.shape[0], angular_bins * 2),\n dtype=bool)\n bin_width = GeneralConstants.PI / angular_bins\n for i in range(angles.shape[0]):\n pred_mask[i, int((angles[i] // bin_width) * 2)] = True\n pred_mask[i, int((angles[i] // bin_width) * 2 + 1)] = True\n\n # Predict with GQ-CNN.\n predictions = gqcnn.predict(image_arr, pose_arr)\n if angular_bins > 0:\n raw_predictions = np.array(predictions)\n predictions = predictions[pred_mask].reshape((-1, 2))\n\n # Aggregate.\n all_predictions.extend(predictions[:, 1].tolist())\n if angular_bins > 0:\n all_predictions_raw.extend(raw_predictions.tolist())\n all_labels.extend(label_arr.tolist())\n\n # Close session.\n gqcnn.close_session()\n\n # Create arrays.\n all_predictions = np.array(all_predictions)\n all_labels = np.array(all_labels)\n train_predictions = all_predictions[train_indices]\n val_predictions = all_predictions[val_indices]\n train_labels = all_labels[train_indices]\n val_labels = all_labels[val_indices]\n if angular_bins > 0:\n all_predictions_raw = np.array(all_predictions_raw)\n train_predictions_raw = all_predictions_raw[train_indices]\n val_predictions_raw = all_predictions_raw[val_indices]\n\n # Aggregate results.\n train_result = BinaryClassificationResult(train_predictions,\n train_labels)\n val_result = BinaryClassificationResult(val_predictions, val_labels)\n train_result.save(os.path.join(model_output_dir, \"train_result.cres\"))\n val_result.save(os.path.join(model_output_dir, \"val_result.cres\"))\n\n # Get stats, plot curves.\n self.logger.info(\"Model %s training error rate: %.3f\" %\n (model_dir, train_result.error_rate))\n self.logger.info(\"Model %s validation error rate: %.3f\" %\n (model_dir, val_result.error_rate))\n\n self.logger.info(\"Model %s training loss: %.3f\" %\n (model_dir, train_result.cross_entropy_loss))\n self.logger.info(\"Model %s validation loss: %.3f\" %\n (model_dir, val_result.cross_entropy_loss))\n\n # Save images.\n vis2d.figure()\n example_dir = os.path.join(model_output_dir, \"examples\")\n if not os.path.exists(example_dir):\n os.mkdir(example_dir)\n\n # Train.\n self.logger.info(\"Saving training examples\")\n train_example_dir = os.path.join(example_dir, \"train\")\n if not os.path.exists(train_example_dir):\n os.mkdir(train_example_dir)\n\n # Train TP.\n true_positive_indices = train_result.true_positive_indices\n np.random.shuffle(true_positive_indices)\n true_positive_indices = true_positive_indices[:self.num_vis]\n for i, j in enumerate(true_positive_indices):\n k = train_indices[j]\n datapoint = dataset.datapoint(\n k, field_names=[image_field_name, pose_field_name])\n vis2d.clf()\n if angular_bins > 0:\n self._plot_grasp(datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=train_predictions_raw[j])\n else:\n self._plot_grasp(datapoint, image_field_name, pose_field_name,\n gripper_mode)\n vis2d.title(\n \"Datapoint %d: Pred: %.3f Label: %.3f\" %\n (k, train_result.pred_probs[j], train_result.labels[j]),\n fontsize=self.font_size)\n vis2d.savefig(\n os.path.join(train_example_dir,\n \"true_positive_%03d.png\" % (i)))\n\n # Train FP.\n false_positive_indices = train_result.false_positive_indices\n np.random.shuffle(false_positive_indices)\n false_positive_indices = false_positive_indices[:self.num_vis]\n for i, j in enumerate(false_positive_indices):\n k = train_indices[j]\n datapoint = dataset.datapoint(\n k, field_names=[image_field_name, pose_field_name])\n vis2d.clf()\n if angular_bins > 0:\n self._plot_grasp(datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=train_predictions_raw[j])\n else:\n self._plot_grasp(datapoint, image_field_name, pose_field_name,\n gripper_mode)\n vis2d.title(\n \"Datapoint %d: Pred: %.3f Label: %.3f\" %\n (k, train_result.pred_probs[j], train_result.labels[j]),\n fontsize=self.font_size)\n vis2d.savefig(\n os.path.join(train_example_dir,\n \"false_positive_%03d.png\" % (i)))\n\n # Train TN.\n true_negative_indices = train_result.true_negative_indices\n np.random.shuffle(true_negative_indices)\n true_negative_indices = true_negative_indices[:self.num_vis]\n for i, j in enumerate(true_negative_indices):\n k = train_indices[j]\n datapoint = dataset.datapoint(\n k, field_names=[image_field_name, pose_field_name])\n vis2d.clf()\n if angular_bins > 0:\n self._plot_grasp(datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=train_predictions_raw[j])\n else:\n self._plot_grasp(datapoint, image_field_name, pose_field_name,\n gripper_mode)\n vis2d.title(\n \"Datapoint %d: Pred: %.3f Label: %.3f\" %\n (k, train_result.pred_probs[j], train_result.labels[j]),\n fontsize=self.font_size)\n vis2d.savefig(\n os.path.join(train_example_dir,\n \"true_negative_%03d.png\" % (i)))\n\n # Train TP.\n false_negative_indices = train_result.false_negative_indices\n np.random.shuffle(false_negative_indices)\n false_negative_indices = false_negative_indices[:self.num_vis]\n for i, j in enumerate(false_negative_indices):\n k = train_indices[j]\n datapoint = dataset.datapoint(\n k, field_names=[image_field_name, pose_field_name])\n vis2d.clf()\n if angular_bins > 0:\n self._plot_grasp(datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=train_predictions_raw[j])\n else:\n self._plot_grasp(datapoint, image_field_name, pose_field_name,\n gripper_mode)\n vis2d.title(\n \"Datapoint %d: Pred: %.3f Label: %.3f\" %\n (k, train_result.pred_probs[j], train_result.labels[j]),\n fontsize=self.font_size)\n vis2d.savefig(\n os.path.join(train_example_dir,\n \"false_negative_%03d.png\" % (i)))\n\n # Val.\n self.logger.info(\"Saving validation examples\")\n val_example_dir = os.path.join(example_dir, \"val\")\n if not os.path.exists(val_example_dir):\n os.mkdir(val_example_dir)\n\n # Val TP.\n true_positive_indices = val_result.true_positive_indices\n np.random.shuffle(true_positive_indices)\n true_positive_indices = true_positive_indices[:self.num_vis]\n for i, j in enumerate(true_positive_indices):\n k = val_indices[j]\n datapoint = dataset.datapoint(\n k, field_names=[image_field_name, pose_field_name])\n vis2d.clf()\n if angular_bins > 0:\n self._plot_grasp(datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=val_predictions_raw[j])\n else:\n self._plot_grasp(datapoint, image_field_name, pose_field_name,\n gripper_mode)\n vis2d.title(\"Datapoint %d: Pred: %.3f Label: %.3f\" %\n (k, val_result.pred_probs[j], val_result.labels[j]),\n fontsize=self.font_size)\n vis2d.savefig(\n os.path.join(val_example_dir, \"true_positive_%03d.png\" % (i)))\n\n # Val FP.\n false_positive_indices = val_result.false_positive_indices\n np.random.shuffle(false_positive_indices)\n false_positive_indices = false_positive_indices[:self.num_vis]\n for i, j in enumerate(false_positive_indices):\n k = val_indices[j]\n datapoint = dataset.datapoint(\n k, field_names=[image_field_name, pose_field_name])\n vis2d.clf()\n if angular_bins > 0:\n self._plot_grasp(datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=val_predictions_raw[j])\n else:\n self._plot_grasp(datapoint, image_field_name, pose_field_name,\n gripper_mode)\n vis2d.title(\"Datapoint %d: Pred: %.3f Label: %.3f\" %\n (k, val_result.pred_probs[j], val_result.labels[j]),\n fontsize=self.font_size)\n vis2d.savefig(\n os.path.join(val_example_dir, \"false_positive_%03d.png\" % (i)))\n\n # Val TN.\n true_negative_indices = val_result.true_negative_indices\n np.random.shuffle(true_negative_indices)\n true_negative_indices = true_negative_indices[:self.num_vis]\n for i, j in enumerate(true_negative_indices):\n k = val_indices[j]\n datapoint = dataset.datapoint(\n k, field_names=[image_field_name, pose_field_name])\n vis2d.clf()\n if angular_bins > 0:\n self._plot_grasp(datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=val_predictions_raw[j])\n else:\n self._plot_grasp(datapoint, image_field_name, pose_field_name,\n gripper_mode)\n vis2d.title(\"Datapoint %d: Pred: %.3f Label: %.3f\" %\n (k, val_result.pred_probs[j], val_result.labels[j]),\n fontsize=self.font_size)\n vis2d.savefig(\n os.path.join(val_example_dir, \"true_negative_%03d.png\" % (i)))\n\n # Val TP.\n false_negative_indices = val_result.false_negative_indices\n np.random.shuffle(false_negative_indices)\n false_negative_indices = false_negative_indices[:self.num_vis]\n for i, j in enumerate(false_negative_indices):\n k = val_indices[j]\n datapoint = dataset.datapoint(\n k, field_names=[image_field_name, pose_field_name])\n vis2d.clf()\n if angular_bins > 0:\n self._plot_grasp(datapoint,\n image_field_name,\n pose_field_name,\n gripper_mode,\n angular_preds=val_predictions_raw[j])\n else:\n self._plot_grasp(datapoint, image_field_name, pose_field_name,\n gripper_mode)\n vis2d.title(\"Datapoint %d: Pred: %.3f Label: %.3f\" %\n (k, val_result.pred_probs[j], val_result.labels[j]),\n fontsize=self.font_size)\n vis2d.savefig(\n os.path.join(val_example_dir, \"false_negative_%03d.png\" % (i)))\n\n # Save summary stats.\n train_summary_stats = {\n \"error_rate\": train_result.error_rate,\n \"ap_score\": train_result.ap_score,\n \"auc_score\": train_result.auc_score,\n \"loss\": train_result.cross_entropy_loss\n }\n train_stats_filename = os.path.join(model_output_dir,\n \"train_stats.json\")\n json.dump(train_summary_stats,\n open(train_stats_filename, \"w\"),\n indent=JSON_INDENT,\n sort_keys=True)\n\n val_summary_stats = {\n \"error_rate\": val_result.error_rate,\n \"ap_score\": val_result.ap_score,\n \"auc_score\": val_result.auc_score,\n \"loss\": val_result.cross_entropy_loss\n }\n val_stats_filename = os.path.join(model_output_dir, \"val_stats.json\")\n json.dump(val_summary_stats,\n open(val_stats_filename, \"w\"),\n indent=JSON_INDENT,\n sort_keys=True)\n\n return train_result, val_result\n\n def _plot(self, model_dir, model_output_dir, train_result, val_result):\n \"\"\"Plot analysis curves.\"\"\"\n self.logger.info(\"Plotting\")\n\n _, model_name = os.path.split(model_output_dir)\n\n # Set params.\n colors = [\"g\", \"b\", \"c\", \"y\", \"m\", \"r\"]\n styles = [\"-\", \"--\", \"-.\", \":\", \"-\"]\n\n # PR, ROC.\n vis2d.clf()\n train_result.precision_recall_curve(plot=True,\n line_width=self.line_width,\n color=colors[0],\n style=styles[0],\n label=\"TRAIN\")\n val_result.precision_recall_curve(plot=True,\n line_width=self.line_width,\n color=colors[1],\n style=styles[1],\n label=\"VAL\")\n vis2d.title(\"Precision Recall Curves\", fontsize=self.font_size)\n handles, labels = vis2d.gca().get_legend_handles_labels()\n vis2d.legend(handles, labels, loc=\"best\")\n figname = os.path.join(model_output_dir, \"precision_recall.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n vis2d.clf()\n train_result.roc_curve(plot=True,\n line_width=self.line_width,\n color=colors[0],\n style=styles[0],\n label=\"TRAIN\")\n val_result.roc_curve(plot=True,\n line_width=self.line_width,\n color=colors[1],\n style=styles[1],\n label=\"VAL\")\n vis2d.title(\"Reciever Operating Characteristic\",\n fontsize=self.font_size)\n handles, labels = vis2d.gca().get_legend_handles_labels()\n vis2d.legend(handles, labels, loc=\"best\")\n figname = os.path.join(model_output_dir, \"roc.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n # Plot histogram of prediction errors.\n num_bins = min(self.num_bins, train_result.num_datapoints)\n\n # Train positives.\n pos_ind = np.where(train_result.labels == 1)[0]\n diffs = np.abs(train_result.labels[pos_ind] -\n train_result.pred_probs[pos_ind])\n vis2d.figure()\n utils.histogram(diffs,\n num_bins,\n bounds=(0, 1),\n normalized=False,\n plot=True)\n vis2d.title(\"Error on Positive Training Examples\",\n fontsize=self.font_size)\n vis2d.xlabel(\"Abs Prediction Error\", fontsize=self.font_size)\n vis2d.ylabel(\"Count\", fontsize=self.font_size)\n figname = os.path.join(model_output_dir,\n \"pos_train_errors_histogram.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n # Train negatives.\n neg_ind = np.where(train_result.labels == 0)[0]\n diffs = np.abs(train_result.labels[neg_ind] -\n train_result.pred_probs[neg_ind])\n vis2d.figure()\n utils.histogram(diffs,\n num_bins,\n bounds=(0, 1),\n normalized=False,\n plot=True)\n vis2d.title(\"Error on Negative Training Examples\",\n fontsize=self.font_size)\n vis2d.xlabel(\"Abs Prediction Error\", fontsize=self.font_size)\n vis2d.ylabel(\"Count\", fontsize=self.font_size)\n figname = os.path.join(model_output_dir,\n \"neg_train_errors_histogram.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n # Histogram of validation errors.\n num_bins = min(self.num_bins, val_result.num_datapoints)\n\n # Val positives.\n pos_ind = np.where(val_result.labels == 1)[0]\n diffs = np.abs(val_result.labels[pos_ind] -\n val_result.pred_probs[pos_ind])\n vis2d.figure()\n utils.histogram(diffs,\n num_bins,\n bounds=(0, 1),\n normalized=False,\n plot=True)\n vis2d.title(\"Error on Positive Validation Examples\",\n fontsize=self.font_size)\n vis2d.xlabel(\"Abs Prediction Error\", fontsize=self.font_size)\n vis2d.ylabel(\"Count\", fontsize=self.font_size)\n figname = os.path.join(model_output_dir,\n \"pos_val_errors_histogram.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n # Val negatives.\n neg_ind = np.where(val_result.labels == 0)[0]\n diffs = np.abs(val_result.labels[neg_ind] -\n val_result.pred_probs[neg_ind])\n vis2d.figure()\n utils.histogram(diffs,\n num_bins,\n bounds=(0, 1),\n normalized=False,\n plot=True)\n vis2d.title(\"Error on Negative Validation Examples\",\n fontsize=self.font_size)\n vis2d.xlabel(\"Abs Prediction Error\", fontsize=self.font_size)\n vis2d.ylabel(\"Count\", fontsize=self.font_size)\n figname = os.path.join(model_output_dir,\n \"neg_val_errors_histogram.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n # Losses.\n try:\n train_errors_filename = os.path.join(model_dir,\n GQCNNFilenames.TRAIN_ERRORS)\n val_errors_filename = os.path.join(model_dir,\n GQCNNFilenames.VAL_ERRORS)\n val_iters_filename = os.path.join(model_dir,\n GQCNNFilenames.VAL_ITERS)\n pct_pos_val_filename = os.path.join(model_dir,\n GQCNNFilenames.PCT_POS_VAL)\n train_losses_filename = os.path.join(model_dir,\n GQCNNFilenames.TRAIN_LOSSES)\n\n raw_train_errors = np.load(train_errors_filename)\n val_errors = np.load(val_errors_filename)\n val_iters = np.load(val_iters_filename)\n pct_pos_val = float(val_errors[0])\n if os.path.exists(pct_pos_val_filename):\n pct_pos_val = 100.0 * np.load(pct_pos_val_filename)\n raw_train_losses = np.load(train_losses_filename)\n\n val_errors = np.r_[pct_pos_val, val_errors]\n val_iters = np.r_[0, val_iters]\n\n # Window the training error.\n i = 0\n train_errors = []\n train_losses = []\n train_iters = []\n while i < raw_train_errors.shape[0]:\n train_errors.append(np.mean(raw_train_errors[i:i + WINDOW]))\n train_losses.append(np.mean(raw_train_losses[i:i + WINDOW]))\n train_iters.append(i)\n i += WINDOW\n train_errors = np.array(train_errors)\n train_losses = np.array(train_losses)\n train_iters = np.array(train_iters)\n\n init_val_error = val_errors[0]\n norm_train_errors = train_errors / init_val_error\n norm_val_errors = val_errors / init_val_error\n norm_final_val_error = val_result.error_rate / val_errors[0]\n if pct_pos_val > 0:\n norm_final_val_error = val_result.error_rate / pct_pos_val\n\n vis2d.clf()\n vis2d.plot(train_iters,\n train_errors,\n linewidth=self.line_width,\n color=\"b\")\n vis2d.plot(val_iters,\n val_errors,\n linewidth=self.line_width,\n color=\"g\")\n vis2d.ylim(0, 100)\n vis2d.legend((\"TRAIN (Minibatch)\", \"VAL\"),\n fontsize=self.font_size,\n loc=\"best\")\n vis2d.xlabel(\"Iteration\", fontsize=self.font_size)\n vis2d.ylabel(\"Error Rate\", fontsize=self.font_size)\n vis2d.title(\"Error Rate vs Training Iteration\",\n fontsize=self.font_size)\n figname = os.path.join(model_output_dir,\n \"training_error_rates.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n vis2d.clf()\n vis2d.plot(train_iters, norm_train_errors, linewidth=4, color=\"b\")\n vis2d.plot(val_iters, norm_val_errors, linewidth=4, color=\"g\")\n vis2d.ylim(0, 2.0)\n vis2d.legend((\"TRAIN (Minibatch)\", \"VAL\"),\n fontsize=self.font_size,\n loc=\"best\")\n vis2d.xlabel(\"Iteration\", fontsize=self.font_size)\n vis2d.ylabel(\"Normalized Error Rate\", fontsize=self.font_size)\n vis2d.title(\"Normalized Error Rate vs Training Iteration\",\n fontsize=self.font_size)\n figname = os.path.join(model_output_dir,\n \"training_norm_error_rates.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n train_losses[train_losses > MAX_LOSS] = MAX_LOSS # CAP LOSSES.\n vis2d.clf()\n vis2d.plot(train_iters,\n train_losses,\n linewidth=self.line_width,\n color=\"b\")\n vis2d.ylim(0, 2.0)\n vis2d.xlabel(\"Iteration\", fontsize=self.font_size)\n vis2d.ylabel(\"Loss\", fontsize=self.font_size)\n vis2d.title(\"Training Loss vs Iteration\", fontsize=self.font_size)\n figname = os.path.join(model_output_dir, \"training_losses.png\")\n vis2d.savefig(figname, dpi=self.dpi)\n\n # Log.\n self.logger.info(\"TRAIN\")\n self.logger.info(\"Original error: %.3f\" % (train_errors[0]))\n self.logger.info(\"Final error: %.3f\" % (train_result.error_rate))\n self.logger.info(\"Orig loss: %.3f\" % (train_losses[0]))\n self.logger.info(\"Final loss: %.3f\" % (train_losses[-1]))\n\n self.logger.info(\"VAL\")\n self.logger.info(\"Original error: %.3f\" % (pct_pos_val))\n self.logger.info(\"Final error: %.3f\" % (val_result.error_rate))\n self.logger.info(\"Normalized error: %.3f\" % (norm_final_val_error))\n\n return (train_errors[0], train_result.error_rate, train_losses[0],\n train_losses[-1], pct_pos_val, val_result.error_rate,\n norm_final_val_error)\n except Exception as e:\n self.logger.error(\"Failed to plot training curves!\\n\" + str(e))\n" }, { "path": "gqcnn/grasping/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\"\"\"\nfrom .grasp import Grasp2D, SuctionPoint2D, MultiSuctionPoint2D\nfrom .grasp_quality_function import (GraspQualityFunctionFactory,\n GQCnnQualityFunction)\nfrom .image_grasp_sampler import (ImageGraspSamplerFactory,\n AntipodalDepthImageGraspSampler)\nfrom .constraint_fn import GraspConstraintFnFactory\nfrom .policy import (RobustGraspingPolicy, CrossEntropyRobustGraspingPolicy,\n FullyConvolutionalGraspingPolicyParallelJaw,\n FullyConvolutionalGraspingPolicySuction,\n UniformRandomGraspingPolicy, RgbdImageState, GraspAction)\nfrom .actions import (NoAction, ParallelJawGrasp3D, SuctionGrasp3D,\n MultiSuctionGrasp3D)\n\n__all__ = [\n \"Grasp2D\", \"SuctionPoint2D\", \"MultiSuctionPoint2D\",\n \"GraspQualityFunctionFactory\", \"GQCnnQualityFunction\",\n \"ImageGraspSamplerFactory\", \"AntipodalDepthImageGraspSampler\",\n \"RobustGraspingPolicy\", \"CrossEntropyRobustGraspingPolicy\",\n \"FullyConvolutionalGraspingPolicyParallelJaw\",\n \"FullyConvolutionalGraspingPolicySuction\", \"UniformRandomGraspingPolicy\",\n \"RgbdImageState\", \"GraspAction\", \"GraspConstraintFnFactory\", \"NoAction\",\n \"ParallelJawGrasp3D\", \"SuctionGrasp3D\", \"MultiSuctionGrasp3D\"\n]\n" }, { "path": "gqcnn/grasping/actions.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nAction classes for representing 3D grasp actions.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nfrom abc import ABC, abstractmethod\n\nimport numpy as np\n\nfrom autolab_core import Point\n\nfrom .grasp import Grasp2D, SuctionPoint2D, MultiSuctionPoint2D\n\n\nclass Action(object):\n \"\"\"Base action class.\n\n Attributes\n ----------\n q_value : float\n Grasp quality.\n id : int\n Integer identifier for the action.\n metadata : dict\n Key-value dict of extra data about the action.\n \"\"\"\n\n def __init__(self, q_value=0.0, id=-1, metadata={}):\n self._q_value = q_value\n self._id = id\n self._metadata = metadata\n\n @property\n def q_value(self):\n return self._q_value\n\n @property\n def id(self):\n return self._id\n\n @property\n def metadata(self):\n return self._metadata\n\n\nclass NoAction(Action):\n \"\"\"Proxy for taking no action when none can be found.\"\"\"\n pass\n\n\nclass GraspAction3D(ABC, Action):\n \"\"\"Base grasp class with grasp specified as an end-effector pose.\n\n Attributes\n ----------\n T_grasp_world : :obj:`RigidTransform`\n Pose of the grasp w.r.t. world coordinate frame.\n \"\"\"\n\n def __init__(self, T_grasp_world, q_value=0.0, id=-1, metadata={}):\n self.T_grasp_world = T_grasp_world\n Action.__init__(self, q_value, id, metadata)\n\n @abstractmethod\n def project(self, camera_intr, T_camera_world):\n pass\n\n\nclass ParallelJawGrasp3D(GraspAction3D):\n \"\"\"Grasping with a parallel-jaw gripper.\n\n Attributes\n ----------\n T_grasp_world : :obj:`RigidTransform`\n Pose of the grasp wrt world coordinate frame.\n \"\"\"\n\n def project(self, camera_intr, T_camera_world, gripper_width=0.05):\n # Compute pose of grasp in camera frame.\n T_grasp_camera = T_camera_world.inverse() * self.T_grasp_world\n y_axis_camera = T_grasp_camera.y_axis[:2]\n if np.linalg.norm(y_axis_camera) > 0:\n y_axis_camera = y_axis_camera / np.linalg.norm(y_axis_camera)\n\n # Compute grasp axis rotation in image space.\n rot_grasp_camera = np.arccos(y_axis_camera[0])\n if y_axis_camera[1] < 0:\n rot_grasp_camera = -rot_grasp_camera\n while rot_grasp_camera < 0:\n rot_grasp_camera += 2 * np.pi\n while rot_grasp_camera > 2 * np.pi:\n rot_grasp_camera -= 2 * np.pi\n\n # Compute grasp center in image space.\n t_grasp_camera = T_grasp_camera.translation\n p_grasp_camera = Point(t_grasp_camera, frame=camera_intr.frame)\n u_grasp_camera = camera_intr.project(p_grasp_camera)\n d_grasp_camera = t_grasp_camera[2]\n return Grasp2D(u_grasp_camera,\n rot_grasp_camera,\n d_grasp_camera,\n width=gripper_width,\n camera_intr=camera_intr)\n\n\nclass SuctionGrasp3D(GraspAction3D):\n \"\"\"Grasping with a suction-based gripper.\n\n Attributes\n ----------\n T_grasp_world : :obj:`RigidTransform`\n Pose of the grasp wrt world coordinate frame.\n \"\"\"\n\n def project(self, camera_intr, T_camera_world):\n # Compute pose of grasp in camera frame.\n T_grasp_camera = T_camera_world.inverse() * self.T_grasp_world\n x_axis_camera = T_grasp_camera.x_axis\n\n # Compute grasp center in image space.\n t_grasp_camera = T_grasp_camera.translation\n p_grasp_camera = Point(t_grasp_camera, frame=camera_intr.frame)\n u_grasp_camera = camera_intr.project(p_grasp_camera)\n d_grasp_camera = t_grasp_camera[2]\n return SuctionPoint2D(u_grasp_camera,\n x_axis_camera,\n d_grasp_camera,\n camera_intr=camera_intr)\n\n\nclass MultiSuctionGrasp3D(GraspAction3D):\n \"\"\"Grasping with a multi-cup suction-based gripper.\n\n Attributes\n ----------\n T_grasp_world : :obj:`RigidTransform`\n Pose of the grasp wrt world coordinate frame.\n \"\"\"\n\n def project(self, camera_intr, T_camera_world):\n # Compute pose of grasp in camera frame.\n T_grasp_camera = T_camera_world.inverse() * self.T_grasp_world\n return MultiSuctionPoint2D(T_grasp_camera, camera_intr=camera_intr)\n" }, { "path": "gqcnn/grasping/constraint_fn.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nConstraint functions for grasp sampling.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nfrom abc import ABC, abstractmethod\n\nimport numpy as np\n\n\nclass GraspConstraintFn(ABC):\n \"\"\"Abstract constraint functions for grasp sampling.\"\"\"\n\n def __init__(self, config):\n # Set params.\n self._config = config\n\n def __call__(self, grasp):\n \"\"\"Evaluates whether or not a grasp is valid.\n\n Parameters\n ----------\n grasp : :obj:`Grasp2D`\n Grasp to evaluate.\n\n Returns\n -------\n bool\n True if the grasp satisfies constraints, False otherwise.\n \"\"\"\n return self.satisfies_constraints(grasp)\n\n @abstractmethod\n def satisfies_constraints(self, grasp):\n \"\"\"Evaluates whether or not a grasp is valid.\n\n Parameters\n ----------\n grasp : :obj:`Grasp2D`\n Grasp to evaluate.\n\n Returns\n -------\n bool\n True if the grasp satisfies constraints, False otherwise.\n \"\"\"\n pass\n\n\nclass DiscreteApproachGraspConstraintFn(GraspConstraintFn):\n \"\"\"Constrains the grasp approach direction into a discrete set of\n angles from the world z direction.\"\"\"\n\n def __init__(self, config):\n # Init superclass.\n GraspConstraintFn.__init__(self, config)\n\n self._max_approach_angle = self._config[\"max_approach_angle\"]\n self._angular_tolerance = self._config[\"angular_tolerance\"]\n self._angular_step = self._config[\"angular_step\"]\n self._T_camera_world = self._config[\"camera_pose\"]\n\n def satisfies_constraints(self, grasp):\n \"\"\"Evaluates whether or not a grasp is valid by evaluating the\n angle between the approach axis and the world z direction.\n\n Parameters\n ----------\n grasp : :obj:`Grasp2D`\n Grasp to evaluate.\n\n Returns\n -------\n bool\n True if the grasp satisfies constraints, False otherwise.\n \"\"\"\n # Find grasp angle in world coordinates.\n axis_world = self._T_camera_world.rotation.dot(grasp.approach_axis)\n angle = np.arccos(-axis_world[2])\n\n # Check closest available angle.\n available_angles = np.array([0.0])\n if self._angular_step > 0:\n available_angles = np.arange(start=0.0,\n stop=self._max_approach_angle,\n step=self._angular_step)\n diff = np.abs(available_angles - angle)\n angle_index = np.argmin(diff)\n if diff[angle_index] < self._angular_tolerance:\n return True\n return False\n\n\nclass GraspConstraintFnFactory(object):\n\n @staticmethod\n def constraint_fn(fn_type, config):\n if fn_type == \"none\":\n return None\n elif fn_type == \"discrete_approach_angle\":\n return DiscreteApproachGraspConstraintFn(config)\n else:\n raise ValueError(\n \"Grasp constraint function type {} not supported!\".format(\n fn_type))\n" }, { "path": "gqcnn/grasping/grasp.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nClasses to encapsulate parallel-jaw grasps in image space.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nimport numpy as np\n\nfrom autolab_core import Point, RigidTransform, CameraIntrinsics\n\n\nclass Grasp2D(object):\n \"\"\"Parallel-jaw grasp in image space.\n\n Attributes\n ----------\n center : :obj:`autolab_core.Point`\n Point in image space.\n angle : float\n Grasp axis angle with the camera x-axis.\n depth : float\n Depth of the grasp center in 3D space.\n width : float\n Distance between the jaws in meters.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Frame of reference for camera that the grasp corresponds to.\n contact_points : list of :obj:`numpy.ndarray`\n Pair of contact points in image space.\n contact_normals : list of :obj:`numpy.ndarray`\n Pair of contact normals in image space.\n \"\"\"\n\n def __init__(self,\n center,\n angle=0.0,\n depth=1.0,\n width=0.0,\n camera_intr=None,\n contact_points=None,\n contact_normals=None):\n self.center = center\n self.angle = angle\n self.depth = depth\n self.width = width\n # If `camera_intr` is none use default primesense camera intrinsics.\n if not camera_intr:\n self.camera_intr = CameraIntrinsics(\"primesense_overhead\",\n fx=525,\n fy=525,\n cx=319.5,\n cy=239.5,\n width=640,\n height=480)\n else:\n self.camera_intr = camera_intr\n self.contact_points = contact_points\n self.contact_normals = contact_normals\n\n frame = \"image\"\n if camera_intr is not None:\n frame = camera_intr.frame\n if isinstance(center, np.ndarray):\n self.center = Point(center, frame=frame)\n\n @property\n def axis(self):\n \"\"\"Returns the grasp axis.\"\"\"\n return np.array([np.cos(self.angle), np.sin(self.angle)])\n\n @property\n def approach_axis(self):\n return np.array([0, 0, 1])\n\n @property\n def approach_angle(self):\n \"\"\"The angle between the grasp approach axis and camera optical axis.\n \"\"\"\n return 0.0\n\n @property\n def frame(self):\n \"\"\"The name of the frame of reference for the grasp.\"\"\"\n if self.camera_intr is None:\n\n raise ValueError(\"Must specify camera intrinsics\")\n return self.camera_intr.frame\n\n @property\n def width_px(self):\n \"\"\"Returns the width in pixels.\"\"\"\n if self.camera_intr is None:\n missing_camera_intr_msg = (\"Must specify camera intrinsics to\"\n \" compute gripper width in 3D space.\")\n raise ValueError(missing_camera_intr_msg)\n # Form the jaw locations in 3D space at the given depth.\n p1 = Point(np.array([0, 0, self.depth]), frame=self.frame)\n p2 = Point(np.array([self.width, 0, self.depth]), frame=self.frame)\n\n # Project into pixel space.\n u1 = self.camera_intr.project(p1)\n u2 = self.camera_intr.project(p2)\n return np.linalg.norm(u1.data - u2.data)\n\n @property\n def endpoints(self):\n \"\"\"Returns the grasp endpoints.\"\"\"\n p1 = self.center.data - (self.width_px / 2) * self.axis\n p2 = self.center.data + (self.width_px / 2) * self.axis\n return p1, p2\n\n @property\n def feature_vec(self):\n \"\"\"Returns the feature vector for the grasp.\n\n `v = [p1, p2, depth]` where `p1` and `p2` are the jaw locations in\n image space.\n \"\"\"\n p1, p2 = self.endpoints\n return np.r_[p1, p2, self.depth]\n\n @staticmethod\n def from_feature_vec(v, width=0.0, camera_intr=None):\n \"\"\"Creates a `Grasp2D` instance from a feature vector and additional\n parameters.\n\n Parameters\n ----------\n v : :obj:`numpy.ndarray`\n Feature vector, see `Grasp2D.feature_vec`.\n width : float\n Grasp opening width, in meters.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Frame of reference for camera that the grasp corresponds to.\n \"\"\"\n # Read feature vec.\n p1 = v[:2]\n p2 = v[2:4]\n depth = v[4]\n\n # Compute center and angle.\n center_px = (p1 + p2) // 2\n center = Point(center_px, camera_intr.frame)\n axis = p2 - p1\n if np.linalg.norm(axis) > 0:\n axis = axis / np.linalg.norm(axis)\n if axis[1] > 0:\n angle = np.arccos(axis[0])\n else:\n angle = -np.arccos(axis[0])\n return Grasp2D(center,\n angle,\n depth,\n width=width,\n camera_intr=camera_intr)\n\n def pose(self, grasp_approach_dir=None):\n \"\"\"Computes the 3D pose of the grasp relative to the camera.\n\n If an approach direction is not specified then the camera\n optical axis is used.\n\n Parameters\n ----------\n grasp_approach_dir : :obj:`numpy.ndarray`\n Approach direction for the grasp in camera basis (e.g. opposite to\n table normal).\n\n Returns\n -------\n :obj:`autolab_core.RigidTransform`\n The transformation from the grasp to the camera frame of reference.\n \"\"\"\n # Check intrinsics.\n if self.camera_intr is None:\n raise ValueError(\n \"Must specify camera intrinsics to compute 3D grasp pose\")\n\n # Compute 3D grasp center in camera basis.\n grasp_center_im = self.center.data\n center_px_im = Point(grasp_center_im, frame=self.camera_intr.frame)\n grasp_center_camera = self.camera_intr.deproject_pixel(\n self.depth, center_px_im)\n grasp_center_camera = grasp_center_camera.data\n\n # Compute 3D grasp axis in camera basis.\n grasp_axis_im = self.axis\n grasp_axis_im = grasp_axis_im / np.linalg.norm(grasp_axis_im)\n grasp_axis_camera = np.array([grasp_axis_im[0], grasp_axis_im[1], 0])\n grasp_axis_camera = grasp_axis_camera / np.linalg.norm(\n grasp_axis_camera)\n\n # Convert to 3D pose.\n grasp_rot_camera, _, _ = np.linalg.svd(grasp_axis_camera.reshape(3, 1))\n grasp_x_camera = grasp_approach_dir\n if grasp_approach_dir is None:\n grasp_x_camera = np.array([0, 0, 1]) # Align with camera Z axis.\n grasp_y_camera = grasp_axis_camera\n grasp_z_camera = np.cross(grasp_x_camera, grasp_y_camera)\n grasp_z_camera = grasp_z_camera / np.linalg.norm(grasp_z_camera)\n grasp_y_camera = np.cross(grasp_z_camera, grasp_x_camera)\n grasp_rot_camera = np.array(\n [grasp_x_camera, grasp_y_camera, grasp_z_camera]).T\n if np.linalg.det(grasp_rot_camera) < 0: # Fix reflections due to SVD.\n grasp_rot_camera[:, 0] = -grasp_rot_camera[:, 0]\n T_grasp_camera = RigidTransform(rotation=grasp_rot_camera,\n translation=grasp_center_camera,\n from_frame=\"grasp\",\n to_frame=self.camera_intr.frame)\n return T_grasp_camera\n\n @staticmethod\n def image_dist(g1, g2, alpha=1.0):\n \"\"\"Computes the distance between grasps in image space.\n\n Uses Euclidean distance with alpha weighting of angles\n\n Parameters\n ----------\n g1 : :obj:`Grasp2D`\n First grasp.\n g2 : :obj:`Grasp2D`\n Second grasp.\n alpha : float\n Weight of angle distance (rad to meters).\n\n Returns\n -------\n float\n Distance between grasps.\n \"\"\"\n # Point to point distances.\n point_dist = np.linalg.norm(g1.center.data - g2.center.data)\n\n # Axis distances.\n dot = max(min(np.abs(g1.axis.dot(g2.axis)), 1.0), -1.0)\n axis_dist = np.arccos(dot)\n return point_dist + alpha * axis_dist\n\n\nclass SuctionPoint2D(object):\n \"\"\"Suction grasp in image space.\n\n Attributes\n ----------\n center : :obj:`autolab_core.Point`\n Point in image space.\n axis : :obj:`numpy.ndarray`\n Dormalized 3-vector representing the direction of the suction tip.\n depth : float\n Depth of the suction point in 3D space.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Frame of reference for camera that the suction point corresponds to.\n \"\"\"\n\n def __init__(self, center, axis=None, depth=1.0, camera_intr=None):\n if axis is None:\n axis = np.array([0, 0, 1])\n\n self.center = center\n self.axis = axis\n\n frame = \"image\"\n if camera_intr is not None:\n frame = camera_intr.frame\n if isinstance(center, np.ndarray):\n self.center = Point(center, frame=frame)\n if isinstance(axis, list):\n self.axis = np.array(axis)\n if np.abs(np.linalg.norm(self.axis) - 1.0) > 1e-3:\n raise ValueError(\"Illegal axis. Must be norm 1.\")\n\n self.depth = depth\n # If `camera_intr` is `None` use default primesense camera intrinsics.\n if not camera_intr:\n self.camera_intr = CameraIntrinsics(\"primesense_overhead\",\n fx=525,\n fy=525,\n cx=319.5,\n cy=239.5,\n width=640,\n height=480)\n else:\n self.camera_intr = camera_intr\n\n @property\n def frame(self):\n \"\"\"The name of the frame of reference for the grasp.\"\"\"\n if self.camera_intr is None:\n raise ValueError(\"Must specify camera intrinsics\")\n return self.camera_intr.frame\n\n @property\n def angle(self):\n \"\"\"The angle that the grasp pivot axis makes in image space.\"\"\"\n rotation_axis = np.cross(self.axis, np.array([0, 0, 1]))\n rotation_axis_image = np.array([rotation_axis[0], rotation_axis[1]])\n angle = 0\n if np.linalg.norm(rotation_axis) > 0:\n rotation_axis_image = rotation_axis_image / np.linalg.norm(\n rotation_axis_image)\n angle = np.arccos(rotation_axis_image[0])\n if rotation_axis[1] < 0:\n angle = -angle\n return angle\n\n @property\n def approach_angle(self):\n \"\"\"The angle between the grasp approach axis and camera optical axis.\n \"\"\"\n dot = max(min(self.axis.dot(np.array([0, 0, 1])), 1.0), -1.0)\n return np.arccos(dot)\n\n @property\n def approach_axis(self):\n return self.axis\n\n @property\n def feature_vec(self):\n \"\"\"Returns the feature vector for the suction point.\n\n Note\n ----\n `v = [center, axis, depth]`\n \"\"\"\n return self.center.data\n\n @staticmethod\n def from_feature_vec(v, camera_intr=None, depth=None, axis=None):\n \"\"\"Creates a `SuctionPoint2D` instance from a feature vector and\n additional parameters.\n\n Parameters\n ----------\n v : :obj:`numpy.ndarray`\n Feature vector, see `Grasp2D.feature_vec`.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Frame of reference for camera that the grasp corresponds to.\n depth : float\n Hard-set the depth for the suction grasp.\n axis : :obj:`numpy.ndarray`\n Normalized 3-vector specifying the approach direction.\n \"\"\"\n # Read feature vec.\n center_px = v[:2]\n\n grasp_axis = np.array([0, 0, -1])\n if v.shape[0] > 2 and axis is None:\n grasp_axis = v[2:5]\n grasp_axis = grasp_axis / np.linalg.norm(grasp_axis)\n elif axis is not None:\n grasp_axis = axis\n\n grasp_depth = 0.5\n if v.shape[0] > 5 and depth is None:\n grasp_depth = v[5]\n elif depth is not None:\n grasp_depth = depth\n\n # Compute center and angle.\n center = Point(center_px, camera_intr.frame)\n return SuctionPoint2D(center,\n grasp_axis,\n grasp_depth,\n camera_intr=camera_intr)\n\n def pose(self):\n \"\"\"Computes the 3D pose of the grasp relative to the camera.\n\n Returns\n -------\n :obj:`autolab_core.RigidTransform`\n The transformation from the grasp to the camera frame of reference.\n \"\"\"\n # Check intrinsics.\n if self.camera_intr is None:\n raise ValueError(\n \"Must specify camera intrinsics to compute 3D grasp pose\")\n\n # Compute 3D grasp center in camera basis.\n suction_center_im = self.center.data\n center_px_im = Point(suction_center_im, frame=self.camera_intr.frame)\n suction_center_camera = self.camera_intr.deproject_pixel(\n self.depth, center_px_im)\n suction_center_camera = suction_center_camera.data\n\n # Compute 3D grasp axis in camera basis.\n suction_axis_camera = self.axis\n\n # Convert to 3D pose.\n suction_x_camera = suction_axis_camera\n suction_z_camera = np.array(\n [-suction_x_camera[1], suction_x_camera[0], 0])\n if np.linalg.norm(suction_z_camera) < 1e-12:\n suction_z_camera = np.array([1.0, 0.0, 0.0])\n suction_z_camera = suction_z_camera / np.linalg.norm(suction_z_camera)\n suction_y_camera = np.cross(suction_z_camera, suction_x_camera)\n suction_rot_camera = np.c_[suction_x_camera, suction_y_camera,\n suction_z_camera]\n\n T_suction_camera = RigidTransform(rotation=suction_rot_camera,\n translation=suction_center_camera,\n from_frame=\"grasp\",\n to_frame=self.camera_intr.frame)\n return T_suction_camera\n\n @staticmethod\n def image_dist(g1, g2, alpha=1.0):\n \"\"\"Computes the distance between grasps in image space.\n\n Uses Euclidean distance with alpha weighting of angles.\n\n Parameters\n ----------\n g1 : :obj:`SuctionPoint2D`\n First suction point.\n g2 : :obj:`SuctionPoint2D`\n Second suction point.\n alpha : float\n Weight of angle distance (rad to meters).\n\n Returns\n -------\n float\n Distance between grasps.\n \"\"\"\n # Point to point distances.\n point_dist = np.linalg.norm(g1.center.data - g2.center.data)\n\n # Axis distances.\n dot = max(min(np.abs(g1.axis.dot(g2.axis)), 1.0), -1.0)\n axis_dist = np.arccos(dot)\n\n return point_dist + alpha * axis_dist\n\n\nclass MultiSuctionPoint2D(object):\n \"\"\"Multi-Cup Suction grasp in image space.\n\n Equivalent to projecting a 6D pose to image space.\n\n Attributes\n ----------\n pose : :obj:`autolab_core.RigidTransform`\n Pose in 3D camera space.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Frame of reference for camera that the suction point corresponds to.\n \"\"\"\n\n def __init__(self, pose, camera_intr=None):\n self._pose = pose\n\n # TODO(vsatish): Confirm that this is really not needed.\n # frame = \"image\"\n # if camera_intr is not None:\n # frame = camera_intr.frame\n\n # If `camera_intr` is `None` use default primesense camera intrinsics.\n if not camera_intr:\n self.camera_intr = CameraIntrinsics(\"primesense_overhead\",\n fx=525,\n fy=525,\n cx=319.5,\n cy=239.5,\n width=640,\n height=480)\n else:\n self.camera_intr = camera_intr\n\n def pose(self):\n return self._pose\n\n @property\n def frame(self):\n \"\"\"The name of the frame of reference for the grasp.\"\"\"\n if self.camera_intr is None:\n raise ValueError(\"Must specify camera intrinsics\")\n return self.camera_intr.frame\n\n @property\n def center(self):\n center_camera = Point(self._pose.translation,\n frame=self.camera_intr.frame)\n center_px = self.camera_intr.project(center_camera)\n return center_px\n\n @property\n def axis(self):\n return self._pose.x_axis\n\n @property\n def approach_axis(self):\n return self.axis\n\n @property\n def approach_angle(self):\n \"\"\"The angle between the grasp approach axis and camera optical axis.\n \"\"\"\n dot = max(min(self.axis.dot(np.array([0, 0, 1])), 1.0), -1.0)\n return np.arccos(dot)\n\n @property\n def angle(self):\n g_axis = self._pose.y_axis\n g_axis_im = np.array([g_axis[0], g_axis[1], 0])\n if np.linalg.norm(g_axis_im) == 0:\n return 0\n theta = np.arctan2(g_axis[1], g_axis[0])\n return theta\n\n @property\n def depth(self):\n return self._pose.translation[2]\n\n @property\n def orientation(self):\n x_axis = self.axis\n y_axis = np.array([x_axis[1], -x_axis[0], 0])\n if np.linalg.norm(y_axis) == 0:\n y_axis = np.array([1, 0, 0])\n y_axis = y_axis / np.linalg.norm(y_axis)\n z_axis = np.cross(x_axis, y_axis)\n R = np.array([x_axis, y_axis, z_axis]).T\n delta_R = R.T.dot(self._pose.rotation)\n orientation = np.arccos(delta_R[1, 1])\n if delta_R[1, 2] > 0:\n orientation = 2 * np.pi - orientation\n return orientation\n\n @property\n def feature_vec(self):\n \"\"\"Returns the feature vector for the suction point.\n\n Note\n ----\n `v = [center, axis, depth]`\n \"\"\"\n return np.r_[self.center.data,\n np.cos(self.orientation),\n np.sin(self.orientation)]\n\n @staticmethod\n def from_feature_vec(v,\n camera_intr=None,\n angle=None,\n depth=None,\n axis=None):\n \"\"\"Creates a `SuctionPoint2D` instance from a feature vector and\n additional parameters.\n\n Parameters\n ----------\n v : :obj:`numpy.ndarray`\n Feature vector, see `Grasp2D.feature_vec`.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Frame of reference for camera that the grasp corresponds to.\n depth : float\n Hard-set the depth for the suction grasp.\n axis : :obj:`numpy.ndarray`\n Normalized 3-vector specifying the approach direction.\n \"\"\"\n # Read feature vec.\n center_px = v[:2]\n\n grasp_angle = 0\n if v.shape[0] > 2 and angle is None:\n # grasp_angle = v[2]\n grasp_vec = v[2:]\n grasp_vec = grasp_vec / np.linalg.norm(grasp_vec)\n grasp_angle = np.arctan2(grasp_vec[1], grasp_vec[0])\n elif angle is not None:\n grasp_angle = angle\n\n grasp_axis = np.array([1, 0, 0])\n if axis is not None:\n grasp_axis = axis\n\n grasp_depth = 0.5\n if depth is not None:\n grasp_depth = depth\n\n x_axis = grasp_axis\n y_axis = np.array([grasp_axis[1], -grasp_axis[0], 0])\n if np.linalg.norm(y_axis) == 0:\n y_axis = np.array([1, 0, 0])\n y_axis = y_axis / np.linalg.norm(y_axis)\n z_axis = np.cross(x_axis, y_axis)\n\n R = np.array([x_axis, y_axis, z_axis]).T\n R = R.dot(RigidTransform.x_axis_rotation(grasp_angle))\n t = camera_intr.deproject_pixel(\n grasp_depth, Point(center_px, frame=camera_intr.frame)).data\n T = RigidTransform(rotation=R,\n translation=t,\n from_frame=\"grasp\",\n to_frame=camera_intr.frame)\n\n # Compute center and angle.\n return MultiSuctionPoint2D(T, camera_intr=camera_intr)\n\n @staticmethod\n def image_dist(g1, g2, alpha=1.0):\n \"\"\"Computes the distance between grasps in image space.\n\n Uses Euclidean distance with alpha weighting of angles.\n\n Parameters\n ----------\n g1 : :obj:`SuctionPoint2D`\n First suction point.\n g2 : :obj:`SuctionPoint2D`\n Second suction point.\n alpha : float\n Weight of angle distance (rad to meters).\n\n Returns\n -------\n float\n Distance between grasps.\n \"\"\"\n # Point to point distances.\n point_dist = np.linalg.norm(g1.center.data - g2.center.data)\n\n # Axis distances.\n dot = max(min(np.abs(g1.axis.dot(g2.axis)), 1.0), -1.0)\n axis_dist = np.arccos(dot)\n\n return point_dist + alpha * axis_dist\n" }, { "path": "gqcnn/grasping/grasp_quality_function.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nGrasp quality functions: suction quality function and parallel jaw grasping\nquality fuction.\n\nAuthors\n-------\nJason Liu & Jeff Mahler\n\"\"\"\nfrom abc import ABC, abstractmethod\nfrom time import time\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport autolab_core.utils as utils\nfrom autolab_core import Point, PointCloud, RigidTransform, Logger, DepthImage\n\nfrom ..model import get_gqcnn_model, get_fc_gqcnn_model\nfrom .grasp import SuctionPoint2D\nfrom ..utils import GeneralConstants, GripperMode\n\n\nclass GraspQualityFunction(ABC):\n \"\"\"Abstract grasp quality class.\"\"\"\n\n def __init__(self):\n # Set up logger.\n self._logger = Logger.get_logger(self.__class__.__name__)\n\n def __call__(self, state, actions, params=None):\n \"\"\"Evaluates grasp quality for a set of actions given a state.\"\"\"\n return self.quality(state, actions, params)\n\n @abstractmethod\n def quality(self, state, actions, params=None):\n \"\"\"Evaluates grasp quality for a set of actions given a state.\n\n Parameters\n ----------\n state : :obj:`object`\n State of the world e.g. image.\n actions : :obj:`list`\n List of actions to evaluate e.g. parallel-jaw or suction grasps.\n params : :obj:`dict`\n Optional parameters for the evaluation.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Vector containing the real-valued grasp quality\n for each candidate.\n \"\"\"\n pass\n\n\nclass ZeroGraspQualityFunction(object):\n \"\"\"Null function.\"\"\"\n\n def quality(self, state, actions, params=None):\n \"\"\"Returns zero for all grasps.\n\n Parameters\n ----------\n state : :obj:`object`\n State of the world e.g. image.\n actions : :obj:`list`\n List of actions to evaluate e.g. parallel-jaw or suction grasps.\n params : :obj:`dict`\n Optional parameters for the evaluation.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Vector containing the real-valued grasp quality\n for each candidate.\n \"\"\"\n return 0.0\n\n\nclass ParallelJawQualityFunction(GraspQualityFunction):\n \"\"\"Abstract wrapper class for parallel jaw quality functions (only image\n based metrics for now).\"\"\"\n\n def __init__(self, config):\n GraspQualityFunction.__init__(self)\n\n # Read parameters.\n self._friction_coef = config[\"friction_coef\"]\n self._max_friction_cone_angle = np.arctan(self._friction_coef)\n\n def friction_cone_angle(self, action):\n \"\"\"Compute the angle between the axis and the boundaries of the\n friction cone.\"\"\"\n if action.contact_points is None or action.contact_normals is None:\n invalid_friction_ang_msg = (\"Cannot compute friction cone angle\"\n \" without precomputed contact points\"\n \" and normals.\")\n raise ValueError(invalid_friction_ang_msg)\n dot_prod1 = min(max(action.contact_normals[0].dot(-action.axis), -1.0),\n 1.0)\n angle1 = np.arccos(dot_prod1)\n dot_prod2 = min(max(action.contact_normals[1].dot(action.axis), -1.0),\n 1.0)\n angle2 = np.arccos(dot_prod2)\n return max(angle1, angle2)\n\n def force_closure(self, action):\n \"\"\"Determine if the grasp is in force closure.\"\"\"\n return (self.friction_cone_angle(action) <\n self._max_friction_cone_angle)\n\n\nclass ComForceClosureParallelJawQualityFunction(ParallelJawQualityFunction):\n \"\"\"Measures the distance to the estimated center of mass for antipodal\n parallel-jaw grasps.\"\"\"\n\n def __init__(self, config):\n \"\"\"Create a best-fit planarity suction metric.\"\"\"\n self._antipodality_pctile = config[\"antipodality_pctile\"]\n ParallelJawQualityFunction.__init__(self, config)\n\n def quality(self, state, actions, params=None):\n \"\"\"Given a parallel-jaw grasp, compute the distance to the center of\n mass of the grasped object.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n An RgbdImageState instance that encapsulates rgbd_im, camera_intr,\n segmask, full_observed.\n action: :obj:`Grasp2D`\n A suction grasp in image space that encapsulates center, approach\n direction, depth, camera_intr.\n params: dict\n Stores params used in computing quality.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Array of the quality for each grasp.\n \"\"\"\n # Compute antipodality.\n antipodality_q = [\n ParallelJawQualityFunction.friction_cone_angle(self, action)\n for action in actions\n ]\n\n # Compute object centroid.\n object_com = state.rgbd_im.center\n if state.segmask is not None:\n nonzero_px = state.segmask.nonzero_pixels()\n object_com = np.mean(nonzero_px, axis=0)\n\n # Compute negative SSE from the best fit plane for each grasp.\n antipodality_thresh = abs(\n np.percentile(antipodality_q, 100 - self._antipodality_pctile))\n qualities = []\n max_q = max(state.rgbd_im.height, state.rgbd_im.width)\n for i, action in enumerate(actions):\n q = max_q\n friction_cone_angle = antipodality_q[i]\n force_closure = ParallelJawQualityFunction.force_closure(\n self, action)\n if force_closure or friction_cone_angle < antipodality_thresh:\n grasp_center = np.array([action.center.y, action.center.x])\n\n if state.obj_segmask is not None:\n grasp_obj_id = state.obj_segmask[grasp_center[0],\n grasp_center[1]]\n obj_mask = state.obj_segmask.segment_mask(grasp_obj_id)\n nonzero_px = obj_mask.nonzero_pixels()\n object_com = np.mean(nonzero_px, axis=0)\n\n q = np.linalg.norm(grasp_center - object_com)\n\n if state.obj_segmask is not None and grasp_obj_id == 0:\n q = max_q\n\n q = (np.exp(-q / max_q) - np.exp(-1)) / (1 - np.exp(-1))\n qualities.append(q)\n\n return np.array(qualities)\n\n\nclass SuctionQualityFunction(GraspQualityFunction):\n \"\"\"Abstract wrapper class for suction quality functions (only image based\n metrics for now).\"\"\"\n\n def __init__(self, config):\n GraspQualityFunction.__init(self)\n\n # Read parameters.\n self._window_size = config[\"window_size\"]\n self._sample_rate = config[\"sample_rate\"]\n\n def _points_in_window(self, point_cloud_image, action, segmask=None):\n \"\"\"Retrieve all points on the object in a box of size\n `self._window_size`.\"\"\"\n # Read indices.\n im_shape = point_cloud_image.shape\n i_start = int(max(action.center.y - self._window_size // 2,\n 0)) # TODO: Confirm div.\n j_start = int(max(action.center.x - self._window_size // 2, 0))\n i_end = int(min(i_start + self._window_size, im_shape[0]))\n j_end = int(min(j_start + self._window_size, im_shape[1]))\n step = int(1 / self._sample_rate)\n\n # Read 3D points in the window.\n points = point_cloud_image[i_start:i_end:step, j_start:j_end:step]\n stacked_points = points.reshape(points.shape[0] * points.shape[1], -1)\n\n # Form the matrices for plane-fitting.\n return stacked_points\n\n def _points_to_matrices(self, points):\n \"\"\"Convert a set of 3D points to an A and b matrix for regression.\"\"\"\n A = points[:, [0, 1]]\n ones = np.ones((A.shape[0], 1))\n A = np.hstack((A, ones))\n b = points[:, 2]\n return A, b\n\n def _best_fit_plane(self, A, b):\n \"\"\"Find a best-fit plane of points.\"\"\"\n try:\n w, _, _, _ = np.linalg.lstsq(A, b)\n except np.linalg.LinAlgError:\n self._logger.warning(\"Could not find a best-fit plane!\")\n raise\n return w\n\n def _sum_of_squared_residuals(self, w, A, z):\n \"\"\"Returns the sum of squared residuals from the plane.\"\"\"\n return (1.0 / A.shape[0]) * np.square(np.linalg.norm(np.dot(A, w) - z))\n\n\nclass BestFitPlanaritySuctionQualityFunction(SuctionQualityFunction):\n \"\"\"A best-fit planarity suction metric.\"\"\"\n\n def __init__(self, config):\n SuctionQualityFunction.__init__(self, config)\n\n def quality(self, state, actions, params=None):\n \"\"\"Given a suction point, compute a score based on a best-fit 3D plane\n of the neighboring points.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n An RgbdImageState instance that encapsulates rgbd_im, camera_intr,\n segmask, full_observed.\n action: :obj:`SuctionPoint2D`\n A suction grasp in image space that encapsulates center, approach\n direction, depth, camera_intr.\n params: dict\n Stores params used in computing suction quality.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Array of the quality for each grasp.\n \"\"\"\n qualities = []\n\n # Deproject points.\n point_cloud_image = state.camera_intr.deproject_to_image(\n state.rgbd_im.depth)\n\n # Compute negative SSE from the best fit plane for each grasp.\n for i, action in enumerate(actions):\n if not isinstance(action, SuctionPoint2D):\n not_suction_msg = (\"This function can only be used to evaluate\"\n \" suction quality.\")\n raise ValueError(not_suction_msg)\n\n # x,y in matrix A and z is vector z.\n points = self._points_in_window(point_cloud_image,\n action,\n segmask=state.segmask)\n A, b = self._points_to_matrices(points)\n # vector w w/ a bias term represents a best-fit plane.\n w = self._best_fit_plane(A, b)\n\n if params is not None and params[\"vis\"][\"plane\"]:\n from visualization import Visualizer2D as vis2d\n from visualization import Visualizer3D as vis3d\n mid_i = A.shape[0] // 2\n pred_z = A.dot(w)\n p0 = np.array([A[mid_i, 0], A[mid_i, 1], pred_z[mid_i]])\n n = np.array([w[0], w[1], -1])\n n = n / np.linalg.norm(n)\n tx = np.array([n[1], -n[0], 0])\n tx = tx / np.linalg.norm(tx)\n ty = np.cross(n, tx)\n R = np.array([tx, ty, n]).T\n T_table_world = RigidTransform(rotation=R,\n translation=p0,\n from_frame=\"patch\",\n to_frame=\"world\")\n\n vis3d.figure()\n vis3d.points(point_cloud_image.to_point_cloud(),\n scale=0.0025,\n subsample=10,\n random=True,\n color=(0, 0, 1))\n vis3d.points(PointCloud(points.T),\n scale=0.0025,\n color=(1, 0, 0))\n vis3d.table(T_table_world, dim=0.01)\n vis3d.show()\n\n vis2d.figure()\n vis2d.imshow(state.rgbd_im.depth)\n vis2d.scatter(action.center.x, action.center.y, s=50, c=\"b\")\n vis2d.show()\n\n # Evaluate how well best-fit plane describles all points in window.\n quality = np.exp(-self._sum_of_squared_residuals(w, A, b))\n qualities.append(quality)\n\n return np.array(qualities)\n\n\nclass ApproachPlanaritySuctionQualityFunction(SuctionQualityFunction):\n \"\"\"A approach planarity suction metric.\"\"\"\n\n def __init__(self, config):\n \"\"\"Create approach planarity suction metric.\"\"\"\n SuctionQualityFunction.__init__(self, config)\n\n def _action_to_plane(self, point_cloud_image, action):\n \"\"\"Convert a plane from point-normal form to general form.\"\"\"\n x = int(action.center.x)\n y = int(action.center.y)\n p_0 = point_cloud_image[y, x]\n n = -action.axis\n # TODO: Confirm divs.\n w = np.array([-n[0] / n[2], -n[1] / n[2], np.dot(n, p_0) / n[2]])\n return w\n\n def quality(self, state, actions, params=None):\n \"\"\"Given a suction point, compute a score based on a best-fit 3D plane\n of the neighboring points.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n An RgbdImageState instance that encapsulates rgbd_im, camera_intr,\n segmask, full_observed.\n action: :obj:`SuctionPoint2D`\n A suction grasp in image space that encapsulates center, approach\n direction, depth, camera_intr.\n params: dict\n Stores params used in computing suction quality.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Array of the quality for each grasp.\n \"\"\"\n qualities = []\n\n # Deproject points.\n point_cloud_image = state.camera_intr.deproject_to_image(\n state.rgbd_im.depth)\n\n # Compute negative SSE from the best fit plane for each grasp.\n for i, action in enumerate(actions):\n if not isinstance(action, SuctionPoint2D):\n not_suction_msg = (\"This function can only be used to evaluate\"\n \" suction quality.\")\n raise ValueError(not_suction_msg)\n\n # x,y in matrix A and z is vector z.\n points = self._points_in_window(point_cloud_image,\n action,\n segmask=state.segmask)\n A, b = self._points_to_matrices(points)\n # vector w w/ a bias term represents a best-fit plane.\n w = self._action_to_plane(point_cloud_image, action)\n\n if params is not None and params[\"vis\"][\"plane\"]:\n from visualization import Visualizer2D as vis2d\n from visualization import Visualizer3D as vis3d\n mid_i = A.shape[0] // 2\n pred_z = A.dot(w)\n p0 = np.array([A[mid_i, 0], A[mid_i, 1], pred_z[mid_i]])\n n = np.array([w[0], w[1], -1])\n n = n / np.linalg.norm(n)\n tx = np.array([n[1], -n[0], 0])\n tx = tx / np.linalg.norm(tx)\n ty = np.cross(n, tx)\n R = np.array([tx, ty, n]).T\n\n c = state.camera_intr.deproject_pixel(action.depth,\n action.center)\n d = Point(c.data - 0.01 * action.axis, frame=c.frame)\n\n T_table_world = RigidTransform(rotation=R,\n translation=p0,\n from_frame=\"patch\",\n to_frame=\"world\")\n\n vis3d.figure()\n vis3d.points(point_cloud_image.to_point_cloud(),\n scale=0.0025,\n subsample=10,\n random=True,\n color=(0, 0, 1))\n vis3d.points(PointCloud(points.T),\n scale=0.0025,\n color=(1, 0, 0))\n vis3d.points(c, scale=0.005, color=(1, 1, 0))\n vis3d.points(d, scale=0.005, color=(1, 1, 0))\n vis3d.table(T_table_world, dim=0.01)\n vis3d.show()\n\n vis2d.figure()\n vis2d.imshow(state.rgbd_im.depth)\n vis2d.scatter(action.center.x, action.center.y, s=50, c=\"b\")\n vis2d.show()\n\n # Evaluate how well best-fit plane describles all points in window.\n quality = np.exp(-self._sum_of_squared_residuals(w, A, b))\n qualities.append(quality)\n\n return np.array(qualities)\n\n\nclass DiscApproachPlanaritySuctionQualityFunction(SuctionQualityFunction):\n \"\"\"A approach planarity suction metric using a disc-shaped window.\"\"\"\n\n def __init__(self, config):\n \"\"\"Create approach planarity suction metric.\"\"\"\n self._radius = config[\"radius\"]\n SuctionQualityFunction.__init__(self, config)\n\n def _action_to_plane(self, point_cloud_image, action):\n \"\"\"Convert a plane from point-normal form to general form.\"\"\"\n x = int(action.center.x)\n y = int(action.center.y)\n p_0 = point_cloud_image[y, x]\n n = -action.axis\n # TODO: Confirm divs.\n w = np.array([-n[0] / n[2], -n[1] / n[2], np.dot(n, p_0) / n[2]])\n return w\n\n def _points_in_window(self, point_cloud_image, action, segmask=None):\n \"\"\"Retrieve all points on the object in a disc of size\n `self._window_size`.\"\"\"\n # Compute plane.\n n = -action.axis\n U, _, _ = np.linalg.svd(n.reshape((3, 1)))\n tangents = U[:, 1:]\n\n # Read indices.\n im_shape = point_cloud_image.shape\n i_start = int(max(action.center.y - self._window_size // 2, 0))\n j_start = int(max(action.center.x - self._window_size // 2, 0))\n i_end = int(min(i_start + self._window_size, im_shape[0]))\n j_end = int(min(j_start + self._window_size, im_shape[1]))\n step = int(1 / self._sample_rate)\n\n # Read 3D points in the window.\n points = point_cloud_image[i_start:i_end:step, j_start:j_end:step]\n stacked_points = points.reshape(points.shape[0] * points.shape[1], -1)\n\n # Compute the center point.\n contact_point = point_cloud_image[int(action.center.y),\n int(action.center.x)]\n\n # Project onto approach plane.\n residuals = stacked_points - contact_point\n coords = residuals.dot(tangents)\n proj_residuals = coords.dot(tangents.T)\n\n # Check distance from the center point along the approach plane.\n dists = np.linalg.norm(proj_residuals, axis=1)\n stacked_points = stacked_points[dists <= self._radius]\n\n # Form the matrices for plane-fitting.\n return stacked_points\n\n def quality(self, state, actions, params=None):\n \"\"\"Given a suction point, compute a score based on a best-fit 3D plane\n of the neighboring points.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n An RgbdImageState instance that encapsulates rgbd_im, camera_intr,\n segmask, full_observed.\n action: :obj:`SuctionPoint2D`\n A suction grasp in image space that encapsulates center,\n approach direction, depth, camera_intr.\n params: dict\n Stores params used in computing suction quality.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Array of the quality for each grasp.\n \"\"\"\n qualities = []\n\n # Deproject points.\n point_cloud_image = state.camera_intr.deproject_to_image(\n state.rgbd_im.depth)\n\n # Compute negative SSE from the best fit plane for each grasp.\n for i, action in enumerate(actions):\n if not isinstance(action, SuctionPoint2D):\n not_suction_msg = (\"This function can only be used to evaluate\"\n \" suction quality.\")\n raise ValueError(not_suction_msg)\n\n # x,y in matrix A and z is vector z.\n points = self._points_in_window(point_cloud_image,\n action,\n segmask=state.segmask)\n A, b = self._points_to_matrices(points)\n # vector w w/ a bias term represents a best-fit plane.\n w = self._action_to_plane(point_cloud_image, action)\n sse = self._sum_of_squared_residuals(w, A, b)\n\n if params is not None and params[\"vis\"][\"plane\"]:\n from visualization import Visualizer2D as vis2d\n from visualization import Visualizer3D as vis3d\n mid_i = A.shape[0] // 2\n pred_z = A.dot(w)\n p0 = np.array([A[mid_i, 0], A[mid_i, 1], pred_z[mid_i]])\n n = np.array([w[0], w[1], -1])\n n = n / np.linalg.norm(n)\n tx = np.array([n[1], -n[0], 0])\n tx = tx / np.linalg.norm(tx)\n ty = np.cross(n, tx)\n R = np.array([tx, ty, n]).T\n\n c = state.camera_intr.deproject_pixel(action.depth,\n action.center)\n d = Point(c.data - 0.01 * action.axis, frame=c.frame)\n\n T_table_world = RigidTransform(rotation=R,\n translation=p0,\n from_frame=\"patch\",\n to_frame=\"world\")\n\n vis3d.figure()\n vis3d.points(point_cloud_image.to_point_cloud(),\n scale=0.0025,\n subsample=10,\n random=True,\n color=(0, 0, 1))\n vis3d.points(PointCloud(points.T),\n scale=0.0025,\n color=(1, 0, 0))\n vis3d.points(c, scale=0.005, color=(1, 1, 0))\n vis3d.points(d, scale=0.005, color=(1, 1, 0))\n vis3d.table(T_table_world, dim=0.01)\n vis3d.show()\n\n vis2d.figure()\n vis2d.imshow(state.rgbd_im.depth)\n vis2d.scatter(action.center.x, action.center.y, s=50, c=\"b\")\n vis2d.show()\n\n # Evaluate how well best-fit plane describles all points in window.\n quality = np.exp(-sse)\n qualities.append(quality)\n\n return np.array(qualities)\n\n\nclass ComApproachPlanaritySuctionQualityFunction(\n ApproachPlanaritySuctionQualityFunction):\n \"\"\"A approach planarity suction metric that ranks sufficiently planar\n points by their distance to the object COM.\"\"\"\n\n def __init__(self, config):\n \"\"\"Create approach planarity suction metric.\"\"\"\n self._planarity_thresh = config[\"planarity_thresh\"]\n\n ApproachPlanaritySuctionQualityFunction.__init__(self, config)\n\n def quality(self, state, actions, params=None):\n \"\"\"Given a suction point, compute a score based on a best-fit 3D plane\n of the neighboring points.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n An RgbdImageState instance that encapsulates rgbd_im, camera_intr,\n segmask, full_observed.\n action: :obj:`SuctionPoint2D`\n A suction grasp in image space that encapsulates center, approach\n direction, depth, camera_intr.\n params: dict\n Stores params used in computing suction quality.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Array of the quality for each grasp.\n \"\"\"\n # Compute planarity.\n sse = ApproachPlanaritySuctionQualityFunction.quality(self,\n state,\n actions,\n params=params)\n\n if params[\"vis\"][\"hist\"]:\n plt.figure()\n utils.histogram(sse,\n 100, (np.min(sse), np.max(sse)),\n normalized=False,\n plot=True)\n plt.show()\n\n # Compute object centroid.\n object_com = state.rgbd_im.center\n if state.segmask is not None:\n nonzero_px = state.segmask.nonzero_pixels()\n object_com = np.mean(nonzero_px, axis=0)\n\n # Threshold.\n qualities = []\n for k, action in enumerate(actions):\n q = max(state.rgbd_im.height, state.rgbd_im.width)\n if np.abs(sse[k]) < self._planarity_thresh:\n grasp_center = np.array([action.center.y, action.center.x])\n q = np.linalg.norm(grasp_center - object_com)\n\n qualities.append(np.exp(-q))\n\n return np.array(qualities)\n\n\nclass ComDiscApproachPlanaritySuctionQualityFunction(\n DiscApproachPlanaritySuctionQualityFunction):\n \"\"\"A approach planarity suction metric that ranks sufficiently planar\n points by their distance to the object COM.\"\"\"\n\n # NOTE: THERE WAS A SLIGHTLY DIFFERENT DUPLICATE ABOVE.\n\n def __init__(self, config):\n \"\"\"Create approach planarity suction metric.\"\"\"\n self._planarity_pctile = config[\"planarity_pctile\"]\n self._planarity_abs_thresh = 0\n if \"planarity_abs_thresh\" in config:\n self._planarity_abs_thresh = np.exp(\n -config[\"planarity_abs_thresh\"])\n\n DiscApproachPlanaritySuctionQualityFunction.__init__(self, config)\n\n def quality(self, state, actions, params=None):\n \"\"\"Given a suction point, compute a score based on a best-fit 3D plane\n of the neighboring points.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n An RgbdImageState instance that encapsulates rgbd_im, camera_intr,\n segmask, full_observed.\n action: :obj:`SuctionPoint2D`\n A suction grasp in image space that encapsulates center, approach\n direction, depth, camera_intr.\n params: dict\n Stores params used in computing suction quality.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Array of the quality for each grasp.\n \"\"\"\n # Compute planarity.\n sse_q = DiscApproachPlanaritySuctionQualityFunction.quality(\n self, state, actions, params=params)\n\n if params[\"vis\"][\"hist\"]:\n plt.figure()\n utils.histogram(sse_q,\n 100, (np.min(sse_q), np.max(sse_q)),\n normalized=False,\n plot=True)\n plt.show()\n\n # Compute object centroid.\n object_com = state.rgbd_im.center\n if state.segmask is not None:\n nonzero_px = state.segmask.nonzero_pixels()\n object_com = np.mean(nonzero_px, axis=0)\n\n # Threshold.\n planarity_thresh = abs(\n np.percentile(sse_q, 100 - self._planarity_pctile))\n qualities = []\n max_q = max(state.rgbd_im.height, state.rgbd_im.width)\n for k, action in enumerate(actions):\n q = max_q\n if sse_q[k] > planarity_thresh or sse_q[\n k] > self._planarity_abs_thresh:\n grasp_center = np.array([action.center.y, action.center.x])\n\n if state.obj_segmask is not None:\n grasp_obj_id = state.obj_segmask[grasp_center[0],\n grasp_center[1]]\n obj_mask = state.obj_segmask.segment_mask(grasp_obj_id)\n nonzero_px = obj_mask.nonzero_pixels()\n object_com = np.mean(nonzero_px, axis=0)\n\n q = np.linalg.norm(grasp_center - object_com)\n\n q = (np.exp(-q / max_q) - np.exp(-1)) / (1 - np.exp(-1))\n qualities.append(q)\n\n return np.array(qualities)\n\n\nclass GaussianCurvatureSuctionQualityFunction(SuctionQualityFunction):\n \"\"\"A approach planarity suction metric.\"\"\"\n\n def __init__(self, config):\n \"\"\"Create approach planarity suction metric.\"\"\"\n SuctionQualityFunction.__init__(self, config)\n\n def _points_to_matrices(self, points):\n \"\"\"Convert a set of 3D points to an A and b matrix for regression.\"\"\"\n x = points[:, 0]\n y = points[:, 1]\n A = np.c_[x, y, x * x, x * y, y * y]\n ones = np.ones([A.shape[0], 1])\n A = np.c_[A, ones]\n b = points[:, 2]\n return A, b\n\n def quality(self, state, actions, params=None):\n \"\"\"Given a suction point, compute a score based on a best-fit 3D plane\n of the neighboring points.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n An RgbdImageState instance that encapsulates rgbd_im, camera_intr,\n segmask, full_observed.\n action: :obj:`SuctionPoint2D`\n A suction grasp in image space that encapsulates center, approach\n direction, depth, camera_intr.\n params: dict\n Stores params used in computing suction quality.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Array of the quality for each grasp.\n \"\"\"\n qualities = []\n\n # Deproject points.\n point_cloud_image = state.camera_intr.deproject_to_image(\n state.rgbd_im.depth)\n\n # Compute negative SSE from the best fit plane for each grasp.\n for i, action in enumerate(actions):\n if not isinstance(action, SuctionPoint2D):\n not_suction_msg = (\"This function can only be used to evaluate\"\n \" suction quality.\")\n raise ValueError(not_suction_msg)\n\n # x,y in matrix A and z is vector z.\n points = self._points_in_window(point_cloud_image,\n action,\n segmask=state.segmask)\n A, b = self._points_to_matrices(points)\n # vector w w/ a bias term represents a best-fit plane.\n w = self._best_fit_plane(A, b)\n\n # Compute curvature.\n fx = w[0]\n fy = w[1]\n fxx = 2 * w[2]\n fxy = w[3]\n fyy = 2 * w[4]\n curvature = (fxx * fyy - fxy**2) / ((1 + fx**2 + fy**2)**2)\n\n # Store quality.\n quality = np.exp(-np.abs(curvature))\n qualities.append(quality)\n\n return np.array(qualities)\n\n\nclass DiscCurvatureSuctionQualityFunction(\n GaussianCurvatureSuctionQualityFunction):\n\n def __init__(self, config):\n \"\"\"Create approach planarity suction metric.\"\"\"\n self._radius = config[\"radius\"]\n SuctionQualityFunction.__init__(self, config)\n\n def _points_in_window(self, point_cloud_image, action, segmask=None):\n \"\"\"Retrieve all points on the object in a disc of size\n `self._window_size`.\"\"\"\n # Read indices.\n im_shape = point_cloud_image.shape\n i_start = int(max(action.center.y - self._window_size // 2, 0))\n j_start = int(max(action.center.x - self._window_size // 2, 0))\n i_end = int(min(i_start + self._window_size, im_shape[0]))\n j_end = int(min(j_start + self._window_size, im_shape[1]))\n step = int(1 / self._sample_rate)\n\n # Read 3D points in the window.\n points = point_cloud_image[i_start:i_end:step, j_start:j_end:step]\n stacked_points = points.reshape(points.shape[0] * points.shape[1], -1)\n\n # Check the distance from the center point.\n contact_point = point_cloud_image[int(action.center.y),\n int(action.center.x)]\n dists = np.linalg.norm(stacked_points - contact_point, axis=1)\n stacked_points = stacked_points[dists <= self._radius]\n\n # Form the matrices for plane-fitting.\n return stacked_points\n\n\nclass ComDiscCurvatureSuctionQualityFunction(\n DiscCurvatureSuctionQualityFunction):\n\n def __init__(self, config):\n \"\"\"Create approach planarity suction metric.\"\"\"\n self._curvature_pctile = config[\"curvature_pctile\"]\n\n DiscCurvatureSuctionQualityFunction.__init__(self, config)\n\n def quality(self, state, actions, params=None):\n \"\"\"Given a suction point, compute a score based on the Gaussian\n curvature.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n An RgbdImageState instance that encapsulates rgbd_im, camera_intr,\n segmask, full_observed.\n action: :obj:`SuctionPoint2D`\n A suction grasp in image space that encapsulates center, approach\n direction, depth, camera_intr.\n params: dict\n Stores params used in computing suction quality.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Array of the quality for each grasp.\n \"\"\"\n # Compute planarity.\n curvature_q = DiscCurvatureSuctionQualityFunction.quality(\n self, state, actions, params=params)\n\n if params[\"vis\"][\"hist\"]:\n plt.figure()\n # NOTE: This used to be an undefined `curvature`.\n utils.histogram(curvature_q,\n 100, (np.min(curvature_q), np.max(curvature_q)),\n normalized=False,\n plot=True)\n plt.show()\n\n # Compute object centroid.\n object_com = state.rgbd_im.center\n if state.segmask is not None:\n nonzero_px = state.segmask.nonzero_pixels()\n object_com = np.mean(nonzero_px, axis=0)\n\n # Threshold.\n curvature_q_thresh = abs(\n np.percentile(curvature_q, 100 - self._curvature_pctile))\n qualities = []\n max_q = max(state.rgbd_im.height, state.rgbd_im.width)\n for k, action in enumerate(actions):\n q = max_q\n if curvature_q[k] > curvature_q_thresh:\n grasp_center = np.array([action.center.y, action.center.x])\n q = np.linalg.norm(grasp_center - object_com)\n\n q = (np.exp(-q / max_q) - np.exp(-1)) / (1 - np.exp(-1))\n qualities.append(q)\n\n return np.array(qualities)\n\n\nclass GQCnnQualityFunction(GraspQualityFunction):\n\n def __init__(self, config):\n \"\"\"Create a GQCNN suction quality function.\"\"\"\n GraspQualityFunction.__init__(self)\n\n # Store parameters.\n self._config = config\n self._gqcnn_model_dir = config[\"gqcnn_model\"]\n self._crop_height = config[\"crop_height\"]\n self._crop_width = config[\"crop_width\"]\n\n # Init GQ-CNN\n self._gqcnn = get_gqcnn_model().load(self._gqcnn_model_dir)\n\n # Open Tensorflow session for gqcnn.\n self._gqcnn.open_session()\n\n def __del__(self):\n try:\n self._gqcnn.close_session()\n except Exception:\n # TODO(vsatish): Except specific exception.\n pass\n\n @property\n def gqcnn(self):\n \"\"\"Returns the GQ-CNN.\"\"\"\n return self._gqcnn\n\n @property\n def gqcnn_recep_height(self):\n return self._gqcnn.im_height\n\n @property\n def gqcnn_recep_width(self):\n return self._gqcnn.im_width\n\n @property\n def gqcnn_stride(self):\n return self._gqcnn.stride\n\n @property\n def config(self):\n \"\"\"Returns the GQCNN quality function parameters.\"\"\"\n return self._config\n\n def grasps_to_tensors(self, grasps, state):\n \"\"\"Converts a list of grasps to an image and pose tensor\n for fast grasp quality evaluation.\n\n Attributes\n ----------\n grasps : :obj:`list` of :obj:`object`\n List of image grasps to convert.\n state : :obj:`RgbdImageState`\n RGB-D image to plan grasps on.\n\n Returns\n -------\n image_arr : :obj:`numpy.ndarray`\n 4D numpy tensor of image to be predicted.\n pose_arr : :obj:`numpy.ndarray`\n 2D numpy tensor of depth values.\n \"\"\"\n # Parse params.\n gqcnn_im_height = self.gqcnn.im_height\n gqcnn_im_width = self.gqcnn.im_width\n gqcnn_num_channels = self.gqcnn.num_channels\n gqcnn_pose_dim = self.gqcnn.pose_dim\n gripper_mode = self.gqcnn.gripper_mode\n num_grasps = len(grasps)\n depth_im = state.rgbd_im.depth\n\n # Allocate tensors.\n tensor_start = time()\n image_tensor = np.zeros(\n [num_grasps, gqcnn_im_height, gqcnn_im_width, gqcnn_num_channels])\n pose_tensor = np.zeros([num_grasps, gqcnn_pose_dim])\n scale = gqcnn_im_height / self._crop_height\n depth_im_scaled = depth_im.resize(scale)\n for i, grasp in enumerate(grasps):\n translation = scale * np.array([\n depth_im.center[0] - grasp.center.data[1],\n depth_im.center[1] - grasp.center.data[0]\n ])\n im_tf = depth_im_scaled\n im_tf = depth_im_scaled.transform(translation, grasp.angle)\n im_tf = im_tf.crop(gqcnn_im_height, gqcnn_im_width)\n image_tensor[i, ...] = im_tf.raw_data\n\n if gripper_mode == GripperMode.PARALLEL_JAW:\n pose_tensor[i] = grasp.depth\n elif gripper_mode == GripperMode.SUCTION:\n pose_tensor[i, ...] = np.array(\n [grasp.depth, grasp.approach_angle])\n elif gripper_mode == GripperMode.MULTI_SUCTION:\n pose_tensor[i] = grasp.depth\n elif gripper_mode == GripperMode.LEGACY_PARALLEL_JAW:\n pose_tensor[i] = grasp.depth\n elif gripper_mode == GripperMode.LEGACY_SUCTION:\n pose_tensor[i, ...] = np.array(\n [grasp.depth, grasp.approach_angle])\n else:\n raise ValueError(\"Gripper mode %s not supported\" %\n (gripper_mode))\n self._logger.debug(\"Tensor conversion took %.3f sec\" %\n (time() - tensor_start))\n return image_tensor, pose_tensor\n\n def quality(self, state, actions, params):\n \"\"\"Evaluate the quality of a set of actions according to a GQ-CNN.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n State of the world described by an RGB-D image.\n actions: :obj:`object`\n Set of grasping actions to evaluate.\n params: dict\n Optional parameters for quality evaluation.\n\n Returns\n -------\n :obj:`list` of float\n Real-valued grasp quality predictions for each\n action, between 0 and 1.\n \"\"\"\n # Form tensors.\n tensor_start = time()\n image_tensor, pose_tensor = self.grasps_to_tensors(actions, state)\n self._logger.info(\"Image transformation took %.3f sec\" %\n (time() - tensor_start))\n if params is not None and params[\"vis\"][\"tf_images\"]:\n # Read vis params.\n k = params[\"vis\"][\"k\"]\n d = utils.sqrt_ceil(k)\n\n # Display grasp transformed images.\n from visualization import Visualizer2D as vis2d\n vis2d.figure(size=(GeneralConstants.FIGSIZE,\n GeneralConstants.FIGSIZE))\n for i, image_tf in enumerate(image_tensor[:k, ...]):\n depth = pose_tensor[i][0]\n vis2d.subplot(d, d, i + 1)\n vis2d.imshow(DepthImage(image_tf))\n vis2d.title(\"Image %d: d=%.3f\" % (i, depth))\n vis2d.show()\n\n # Predict grasps.\n predict_start = time()\n output_arr = self.gqcnn.predict(image_tensor, pose_tensor)\n q_values = output_arr[:, -1]\n self._logger.info(\"Inference took %.3f sec\" % (time() - predict_start))\n return q_values.tolist()\n\n\nclass NoMagicQualityFunction(GraspQualityFunction):\n\n def __init__(self, config):\n \"\"\"Create a quality that uses `nomagic_net` as a quality function.\"\"\"\n from nomagic_submission import ConvNetModel\n from tensorpack import SaverRestore\n from tensorpack.predict import OfflinePredictor\n from tensorpack.predict.config import PredictConfig\n\n GraspQualityFunction.__init(self)\n\n # Store parameters.\n self._model_path = config[\"gqcnn_model\"]\n self._batch_size = config[\"batch_size\"]\n self._crop_height = config[\"crop_height\"]\n self._crop_width = config[\"crop_width\"]\n self._im_height = config[\"im_height\"]\n self._im_width = config[\"im_width\"]\n self._num_channels = config[\"num_channels\"]\n self._pose_dim = config[\"pose_dim\"]\n self._gripper_mode = config[\"gripper_mode\"]\n self._data_mean = config[\"data_mean\"]\n self._data_std = config[\"data_std\"]\n\n # Init config.\n model = ConvNetModel()\n self._config = PredictConfig(model=model,\n session_init=SaverRestore(\n self._model_path),\n output_names=[\"prob\"])\n self._predictor = OfflinePredictor(self._config)\n\n @property\n def gqcnn(self):\n \"\"\"Returns the GQ-CNN.\"\"\"\n return self._predictor\n\n @property\n def config(self):\n \"\"\"Returns the GQCNN suction quality function parameters.\"\"\"\n return self._config\n\n def grasps_to_tensors(self, grasps, state):\n \"\"\"Converts a list of grasps to an image and pose tensor\n for fast grasp quality evaluation.\n\n Attributes\n ----------\n grasps : :obj:`list` of :obj:`object`\n List of image grasps to convert.\n state : :obj:`RgbdImageState`\n RGB-D image to plan grasps on.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n 4D numpy tensor of image to be predicted.\n :obj:`numpy.ndarray`\n 2D numpy tensor of depth values.\n \"\"\"\n # Parse params.\n gqcnn_im_height = self._im_height\n gqcnn_im_width = self._im_width\n gqcnn_num_channels = self._num_channels\n gqcnn_pose_dim = self._pose_dim\n gripper_mode = self._gripper_mode\n num_grasps = len(grasps)\n depth_im = state.rgbd_im.depth\n\n # Allocate tensors.\n tensor_start = time()\n image_tensor = np.zeros(\n [num_grasps, gqcnn_im_height, gqcnn_im_width, gqcnn_num_channels])\n pose_tensor = np.zeros([num_grasps, gqcnn_pose_dim])\n scale = gqcnn_im_height / self._crop_height\n depth_im_scaled = depth_im.resize(scale)\n for i, grasp in enumerate(grasps):\n translation = scale * np.array([\n depth_im.center[0] - grasp.center.data[1],\n depth_im.center[1] - grasp.center.data[0]\n ])\n im_tf = depth_im_scaled\n im_tf = depth_im_scaled.transform(translation, grasp.angle)\n im_tf = im_tf.crop(gqcnn_im_height, gqcnn_im_width)\n\n im_encoded = cv2.imencode(\".png\", np.uint8(im_tf.raw_data *\n 255))[1].tostring()\n im_decoded = cv2.imdecode(np.frombuffer(im_encoded, np.uint8),\n 0) / 255.0\n image_tensor[i, :, :,\n 0] = ((im_decoded - self._data_mean) / self._data_std)\n\n if gripper_mode == GripperMode.PARALLEL_JAW:\n pose_tensor[i] = grasp.depth\n elif gripper_mode == GripperMode.SUCTION:\n pose_tensor[i, ...] = np.array(\n [grasp.depth, grasp.approach_angle])\n elif gripper_mode == GripperMode.LEGACY_PARALLEL_JAW:\n pose_tensor[i] = grasp.depth\n elif gripper_mode == GripperMode.LEGACY_SUCTION:\n pose_tensor[i, ...] = np.array(\n [grasp.depth, grasp.approach_angle])\n else:\n raise ValueError(\"Gripper mode %s not supported\" %\n (gripper_mode))\n self._logger.debug(\"Tensor conversion took %.3f sec\" %\n (time() - tensor_start))\n return image_tensor, pose_tensor\n\n def quality(self, state, actions, params):\n \"\"\"Evaluate the quality of a set of actions according to a GQ-CNN.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n State of the world described by an RGB-D image.\n actions: :obj:`object`\n Set of grasping actions to evaluate.\n params: dict\n Optional parameters for quality evaluation.\n\n Returns\n -------\n :obj:`list` of float\n Real-valued grasp quality predictions for each action, between 0\n and 1.\n \"\"\"\n # Form tensors.\n image_tensor, pose_tensor = self.grasps_to_tensors(actions, state)\n if params is not None and params[\"vis\"][\"tf_images\"]:\n # Read vis params.\n k = params[\"vis\"][\"k\"]\n d = utils.sqrt_ceil(k)\n\n # Display grasp transformed images.\n from visualization import Visualizer2D as vis2d\n vis2d.figure(size=(GeneralConstants.FIGSIZE,\n GeneralConstants.FIGSIZE))\n for i, image_tf in enumerate(image_tensor[:k, ...]):\n depth = pose_tensor[i][0]\n vis2d.subplot(d, d, i + 1)\n vis2d.imshow(DepthImage(image_tf))\n vis2d.title(\"Image %d: d=%.3f\" % (i, depth))\n vis2d.show()\n\n # Predict grasps.\n num_actions = len(actions)\n null_arr = -1 * np.ones(self._batch_size)\n predict_start = time()\n output_arr = np.zeros([num_actions, 2])\n cur_i = 0\n end_i = cur_i + min(self._batch_size, num_actions - cur_i)\n while cur_i < num_actions:\n output_arr[cur_i:end_i, :] = self.gqcnn(\n image_tensor[cur_i:end_i, :, :, 0],\n pose_tensor[cur_i:end_i, 0], null_arr)[0]\n cur_i = end_i\n end_i = cur_i + min(self._batch_size, num_actions - cur_i)\n q_values = output_arr[:, -1]\n self._logger.debug(\"Prediction took %.3f sec\" %\n (time() - predict_start))\n return q_values.tolist()\n\n\nclass FCGQCnnQualityFunction(GraspQualityFunction):\n\n def __init__(self, config):\n \"\"\"Grasp quality function using the fully-convolutional gqcnn.\"\"\"\n GraspQualityFunction.__init__(self)\n\n # Store parameters.\n self._config = config\n self._model_dir = config[\"gqcnn_model\"]\n self._backend = config[\"gqcnn_backend\"]\n self._fully_conv_config = config[\"fully_conv_gqcnn_config\"]\n\n # Init fcgqcnn.\n self._fcgqcnn = get_fc_gqcnn_model(backend=self._backend).load(\n self._model_dir, self._fully_conv_config)\n\n # Open Tensorflow session for fcgqcnn.\n self._fcgqcnn.open_session()\n\n def __del__(self):\n try:\n self._fcgqcnn.close_session()\n except Exception:\n # TODO(vsatish): Except specific exception.\n pass\n\n @property\n def gqcnn(self):\n \"\"\"Returns the FC-GQCNN.\"\"\"\n return self._fcgqcnn\n\n @property\n def config(self):\n \"\"\"Returns the FC-GQCNN quality function parameters.\"\"\"\n return self._config\n\n def quality(self, images, depths, params=None):\n return self._fcgqcnn.predict(images, depths)\n\n\nclass GraspQualityFunctionFactory(object):\n \"\"\"Factory for grasp quality functions.\"\"\"\n\n @staticmethod\n def quality_function(metric_type, config):\n if metric_type == \"zero\":\n return ZeroGraspQualityFunction()\n elif metric_type == \"parallel_jaw_com_force_closure\":\n return ComForceClosureParallelJawQualityFunction(config)\n elif metric_type == \"suction_best_fit_planarity\":\n return BestFitPlanaritySuctionQualityFunction(config)\n elif metric_type == \"suction_approach_planarity\":\n return ApproachPlanaritySuctionQualityFunction(config)\n elif metric_type == \"suction_com_approach_planarity\":\n return ComApproachPlanaritySuctionQualityFunction(config)\n elif metric_type == \"suction_disc_approach_planarity\":\n return DiscApproachPlanaritySuctionQualityFunction(config)\n elif metric_type == \"suction_com_disc_approach_planarity\":\n return ComDiscApproachPlanaritySuctionQualityFunction(config)\n elif metric_type == \"suction_gaussian_curvature\":\n return GaussianCurvatureSuctionQualityFunction(config)\n elif metric_type == \"suction_disc_curvature\":\n return DiscCurvatureSuctionQualityFunction(config)\n elif metric_type == \"suction_com_disc_curvature\":\n return ComDiscCurvatureSuctionQualityFunction(config)\n elif metric_type == \"gqcnn\":\n return GQCnnQualityFunction(config)\n elif metric_type == \"nomagic\":\n return NoMagicQualityFunction(config)\n elif metric_type == \"fcgqcnn\":\n return FCGQCnnQualityFunction(config)\n else:\n raise ValueError(\"Grasp function type %s not supported!\" %\n (metric_type))\n" }, { "path": "gqcnn/grasping/image_grasp_sampler.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nClasses for sampling a set of grasps directly from images to generate data for\na neural network.\n\nAuthor\n------\nJeff Mahler & Sherdil Niyaz\n\"\"\"\nfrom abc import ABC, abstractmethod\nimport random\nfrom time import time\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport scipy.ndimage.filters as snf\nimport scipy.spatial.distance as ssd\nimport scipy.stats as ss\n\nfrom autolab_core import (Point, RigidTransform, Logger,\n DepthImage, RgbdImage, GdImage)\nfrom visualization import Visualizer2D as vis\n\nfrom .grasp import Grasp2D, SuctionPoint2D, MultiSuctionPoint2D\n\n\ndef force_closure(p1, p2, n1, n2, mu):\n \"\"\"Computes whether or not the point and normal pairs are in force\n closure.\"\"\"\n # Line between the contacts.\n v = p2 - p1\n v = v / np.linalg.norm(v)\n\n # Compute cone membership.\n alpha = np.arctan(mu)\n dot_1 = max(min(n1.dot(-v), 1.0), -1.0)\n dot_2 = max(min(n2.dot(v), 1.0), -1.0)\n in_cone_1 = (np.arccos(dot_1) < alpha)\n in_cone_2 = (np.arccos(dot_2) < alpha)\n return (in_cone_1 and in_cone_2)\n\n\nclass DepthSamplingMode(object):\n \"\"\"Modes for sampling grasp depth.\"\"\"\n UNIFORM = \"uniform\"\n MIN = \"min\"\n MAX = \"max\"\n\n\nclass ImageGraspSampler(ABC):\n \"\"\"Wraps image to crane grasp candidate generation for easy deployment of\n GQ-CNN.\n\n Attributes\n ----------\n config : :obj:`autolab_core.YamlConfig`\n A dictionary-like object containing the parameters of the sampler.\n \"\"\"\n\n def __init__(self, config):\n # Set params.\n self._config = config\n\n # Setup logger.\n self._logger = Logger.get_logger(self.__class__.__name__)\n\n def sample(self,\n rgbd_im,\n camera_intr,\n num_samples,\n segmask=None,\n seed=None,\n visualize=False,\n constraint_fn=None):\n \"\"\"Samples a set of 2D grasps from a given RGB-D image.\n\n Parameters\n ----------\n rgbd_im : :obj:`autolab_core.RgbdImage`\n RGB-D image to sample from.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the camera that captured the images.\n num_samples : int\n Number of grasps to sample.\n segmask : :obj:`autolab_core.BinaryImage`\n Binary image segmenting out the object of interest.\n seed : int\n Number to use in random seed (`None` if no seed).\n visualize : bool\n Whether or not to show intermediate samples (for debugging).\n constraint_fn : :obj:`GraspConstraintFn`\n Constraint function to apply to grasps.\n\n Returns\n -------\n :obj:`list` of :obj:`Grasp2D`\n The list of grasps in image space.\n \"\"\"\n # Set random seed for determinism.\n if seed is not None:\n random.seed(seed)\n np.random.seed(seed)\n\n # Sample an initial set of grasps (without depth).\n self._logger.debug(\"Sampling 2d candidates\")\n sampling_start = time()\n grasps = self._sample(rgbd_im,\n camera_intr,\n num_samples,\n segmask=segmask,\n visualize=visualize,\n constraint_fn=constraint_fn)\n sampling_stop = time()\n self._logger.debug(\"Sampled %d grasps from image\" % (len(grasps)))\n self._logger.debug(\"Sampling grasps took %.3f sec\" %\n (sampling_stop - sampling_start))\n return grasps\n\n @abstractmethod\n def _sample(self,\n rgbd_im,\n camera_intr,\n num_samples,\n segmask=None,\n visualize=False,\n constraint_fn=None):\n \"\"\"Sample a set of 2D grasp candidates from a depth image.\n\n Subclasses must override.\n\n Parameters\n ----------\n rgbd_im : :obj:`autolab_core.RgbdImage`\n RGB-D image to sample from.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the camera that captured the images.\n num_samples : int\n Number of grasps to sample.\n segmask : :obj:`autolab_core.BinaryImage`\n Binary image segmenting out the object of interest.\n visualize : bool\n Whether or not to show intermediate samples (for debugging).\n constraint_fn : :obj:`GraspConstraintFn`\n Constraint function to apply to grasps.\n\n Returns\n -------\n :obj:`list` of :obj:`Grasp2D`\n List of 2D grasp candidates.\n \"\"\"\n pass\n\n\nclass AntipodalDepthImageGraspSampler(ImageGraspSampler):\n \"\"\"Grasp sampler for antipodal point pairs from depth image gradients.\n\n Notes\n -----\n Required configuration parameters are specified in Other Parameters.\n\n Other Parameters\n ----------------\n gripper_width : float\n Width of the gripper, in meters.\n friction_coef : float\n Friction coefficient for 2D force closure.\n depth_grad_thresh : float\n Threshold for depth image gradients to determine edge points for\n sampling.\n depth_grad_gaussian_sigma : float\n Sigma used for pre-smoothing the depth image for better gradients.\n downsample_rate : float\n Factor to downsample the depth image by before sampling grasps.\n max_rejection_samples : int\n Ceiling on the number of grasps to check in antipodal grasp rejection\n sampling.\n max_dist_from_center : int\n Maximum allowable distance of a grasp from the image center.\n min_grasp_dist : float\n Threshold on the grasp distance.\n angle_dist_weight : float\n Amount to weight the angle difference in grasp distance computation.\n depth_samples_per_grasp : int\n Number of depth samples to take per grasp.\n min_depth_offset : float\n Offset from the minimum depth at the grasp center pixel to use in depth\n sampling.\n max_depth_offset : float\n Offset from the maximum depth across all edges.\n depth_sample_win_height : float\n Height of a window around the grasp center pixel used to determine min\n depth.\n depth_sample_win_height : float\n Width of a window around the grasp center pixel used to determine min\n depth.\n depth_sampling_mode : str\n Name of depth sampling mode (uniform, min, max).\n \"\"\"\n\n def __init__(self, config, gripper_width=np.inf):\n # Init superclass.\n ImageGraspSampler.__init__(self, config)\n\n # Antipodality params.\n self._gripper_width = self._config[\"gripper_width\"]\n self._friction_coef = self._config[\"friction_coef\"]\n self._depth_grad_thresh = self._config[\"depth_grad_thresh\"]\n self._depth_grad_gaussian_sigma = self._config[\n \"depth_grad_gaussian_sigma\"]\n self._downsample_rate = self._config[\"downsample_rate\"]\n self._rescale_factor = 1.0 / self._downsample_rate\n self._max_rejection_samples = self._config[\"max_rejection_samples\"]\n\n self._min_num_edge_pixels = 0\n if \"min_num_edge_pixels\" in self._config:\n self._min_num_edge_pixels = self._config[\"min_num_edge_pixels\"]\n\n # Distance thresholds for rejection sampling.\n self._max_dist_from_center = self._config[\"max_dist_from_center\"]\n self._min_dist_from_boundary = self._config[\"min_dist_from_boundary\"]\n self._min_grasp_dist = self._config[\"min_grasp_dist\"]\n self._angle_dist_weight = self._config[\"angle_dist_weight\"]\n\n # Depth sampling params.\n self._depth_samples_per_grasp = max(\n self._config[\"depth_samples_per_grasp\"], 1)\n self._min_depth_offset = self._config[\"min_depth_offset\"]\n self._max_depth_offset = self._config[\"max_depth_offset\"]\n self._h = self._config[\"depth_sample_win_height\"]\n self._w = self._config[\"depth_sample_win_width\"]\n self._depth_sampling_mode = self._config[\"depth_sampling_mode\"]\n\n # Perturbation.\n self._grasp_center_sigma = 0.0\n if \"grasp_center_sigma\" in self._config:\n self._grasp_center_sigma = self._config[\"grasp_center_sigma\"]\n self._grasp_angle_sigma = 0.0\n if \"grasp_angle_sigma\" in self._config:\n self._grasp_angle_sigma = np.deg2rad(\n self._config[\"grasp_angle_sigma\"])\n\n def _surface_normals(self, depth_im, edge_pixels):\n \"\"\"Return an array of the surface normals at the edge pixels.\"\"\"\n # Compute the gradients.\n grad = np.gradient(depth_im.data.astype(np.float32))\n\n # Compute surface normals.\n normals = np.zeros([edge_pixels.shape[0], 2])\n for i, pix in enumerate(edge_pixels):\n dx = grad[1][pix[0], pix[1]]\n dy = grad[0][pix[0], pix[1]]\n normal_vec = np.array([dy, dx])\n if np.linalg.norm(normal_vec) == 0:\n normal_vec = np.array([1, 0])\n normal_vec = normal_vec / np.linalg.norm(normal_vec)\n normals[i, :] = normal_vec\n\n return normals\n\n def _sample_depth(self, min_depth, max_depth):\n \"\"\"Samples a depth value between the min and max.\"\"\"\n depth_sample = max_depth\n if self._depth_sampling_mode == DepthSamplingMode.UNIFORM:\n depth_sample = min_depth + (max_depth -\n min_depth) * np.random.rand()\n elif self._depth_sampling_mode == DepthSamplingMode.MIN:\n depth_sample = min_depth\n return depth_sample\n\n def _sample(self,\n image,\n camera_intr,\n num_samples,\n segmask=None,\n visualize=False,\n constraint_fn=None):\n \"\"\"Sample a set of 2D grasp candidates from a depth image.\n\n Parameters\n ----------\n image : :obj:`autolab_core.RgbdImage` or :obj:`autolab_core.DepthImage` or :obj:`autolab_core.GdImage` # noqa: E501\n RGB-D or Depth image to sample from.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the camera that captured the images.\n num_samples : int\n Number of grasps to sample\n segmask : :obj:`autolab_core.BinaryImage`\n Binary image segmenting out the object of interest.\n visualize : bool\n Whether or not to show intermediate samples (for debugging).\n constraint_fn : :obj:`GraspConstraintFn`\n Constraint function to apply to grasps.\n\n Returns\n -------\n :obj:`list` of :obj:`Grasp2D`\n List of 2D grasp candidates.\n \"\"\"\n if isinstance(image, RgbdImage) or isinstance(image, GdImage):\n depth_im = image.depth\n elif isinstance(image, DepthImage):\n depth_im = image\n else:\n raise ValueError(\n \"image type must be one of [RgbdImage, DepthImage, GdImage]\")\n\n # Sample antipodal pairs in image space.\n grasps = self._sample_antipodal_grasps(depth_im,\n camera_intr,\n num_samples,\n segmask=segmask,\n visualize=visualize,\n constraint_fn=constraint_fn)\n return grasps\n\n def _sample_antipodal_grasps(self,\n depth_im,\n camera_intr,\n num_samples,\n segmask=None,\n visualize=False,\n constraint_fn=None):\n \"\"\"Sample a set of 2D grasp candidates from a depth image by finding\n depth edges, then uniformly sampling point pairs and keeping only\n antipodal grasps with width less than the maximum allowable.\n\n Parameters\n ----------\n depth_im : :obj:\"autolab_core.DepthImage\"\n Depth image to sample from.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the camera that captured the images.\n num_samples : int\n Number of grasps to sample.\n segmask : :obj:`autolab_core.BinaryImage`\n Binary image segmenting out the object of interest.\n visualize : bool\n Whether or not to show intermediate samples (for debugging).\n constraint_fn : :obj:`GraspConstraintFn`\n Constraint function to apply to grasps.\n\n Returns\n -------\n :obj:`list` of :obj:`Grasp2D`\n List of 2D grasp candidates.\n \"\"\"\n # Compute edge pixels.\n edge_start = time()\n depth_im = depth_im.apply(snf.gaussian_filter,\n sigma=self._depth_grad_gaussian_sigma)\n scale_factor = self._rescale_factor\n depth_im_downsampled = depth_im.resize(scale_factor)\n depth_im_threshed = depth_im_downsampled.threshold_gradients(\n self._depth_grad_thresh)\n edge_pixels = (1.0 / scale_factor) * depth_im_threshed.zero_pixels()\n edge_pixels = edge_pixels.astype(np.int16)\n\n depth_im_mask = depth_im.copy()\n if segmask is not None:\n edge_pixels = np.array(\n [p for p in edge_pixels if np.any(segmask[p[0], p[1]] > 0)])\n depth_im_mask = depth_im.mask_binary(segmask)\n\n # Re-threshold edges if there are too few.\n if edge_pixels.shape[0] < self._min_num_edge_pixels:\n self._logger.info(\"Too few edge pixels!\")\n depth_im_threshed = depth_im.threshold_gradients(\n self._depth_grad_thresh)\n edge_pixels = depth_im_threshed.zero_pixels()\n edge_pixels = edge_pixels.astype(np.int16)\n depth_im_mask = depth_im.copy()\n if segmask is not None:\n edge_pixels = np.array([\n p for p in edge_pixels if np.any(segmask[p[0], p[1]] > 0)\n ])\n depth_im_mask = depth_im.mask_binary(segmask)\n\n num_pixels = edge_pixels.shape[0]\n self._logger.debug(\"Depth edge detection took %.3f sec\" %\n (time() - edge_start))\n self._logger.debug(\"Found %d edge pixels\" % (num_pixels))\n\n # Compute point cloud.\n point_cloud_im = camera_intr.deproject_to_image(depth_im_mask)\n\n # Compute_max_depth.\n depth_data = depth_im_mask.data[depth_im_mask.data > 0]\n if depth_data.shape[0] == 0:\n return []\n\n min_depth = np.min(depth_data) + self._min_depth_offset\n max_depth = np.max(depth_data) + self._max_depth_offset\n\n # Compute surface normals.\n normal_start = time()\n edge_normals = self._surface_normals(depth_im, edge_pixels)\n self._logger.debug(\"Normal computation took %.3f sec\" %\n (time() - normal_start))\n\n if visualize:\n edge_pixels = edge_pixels[::2, :]\n edge_normals = edge_normals[::2, :]\n\n vis.figure()\n vis.subplot(1, 3, 1)\n vis.imshow(depth_im)\n if num_pixels > 0:\n vis.scatter(edge_pixels[:, 1], edge_pixels[:, 0], s=2, c=\"b\")\n\n X = [pix[1] for pix in edge_pixels]\n Y = [pix[0] for pix in edge_pixels]\n U = [3 * pix[1] for pix in edge_normals]\n V = [-3 * pix[0] for pix in edge_normals]\n plt.quiver(X,\n Y,\n U,\n V,\n units=\"x\",\n scale=0.25,\n width=0.5,\n zorder=2,\n color=\"r\")\n vis.title(\"Edge pixels and normals\")\n\n vis.subplot(1, 3, 2)\n vis.imshow(depth_im_threshed)\n vis.title(\"Edge map\")\n\n vis.subplot(1, 3, 3)\n vis.imshow(segmask)\n vis.title(\"Segmask\")\n vis.show()\n\n # Exit if no edge pixels.\n if num_pixels == 0:\n return []\n\n # Form set of valid candidate point pairs.\n pruning_start = time()\n max_grasp_width_px = Grasp2D(Point(np.zeros(2)),\n 0.0,\n min_depth,\n width=self._gripper_width,\n camera_intr=camera_intr).width_px\n normal_ip = edge_normals.dot(edge_normals.T)\n dists = ssd.squareform(ssd.pdist(edge_pixels))\n valid_indices = np.where(\n (normal_ip < -np.cos(np.arctan(self._friction_coef)))\n & (dists < max_grasp_width_px) & (dists > 0.0))\n valid_indices = np.c_[valid_indices[0], valid_indices[1]]\n self._logger.debug(\"Normal pruning %.3f sec\" %\n (time() - pruning_start))\n\n # Raise exception if no antipodal pairs.\n num_pairs = valid_indices.shape[0]\n if num_pairs == 0:\n return []\n\n # Prune out grasps.\n contact_points1 = edge_pixels[valid_indices[:, 0], :]\n contact_points2 = edge_pixels[valid_indices[:, 1], :]\n contact_normals1 = edge_normals[valid_indices[:, 0], :]\n contact_normals2 = edge_normals[valid_indices[:, 1], :]\n v = contact_points1 - contact_points2\n v_norm = np.linalg.norm(v, axis=1)\n v = v / np.tile(v_norm[:, np.newaxis], [1, 2])\n ip1 = np.sum(contact_normals1 * v, axis=1)\n ip2 = np.sum(contact_normals2 * (-v), axis=1)\n ip1[ip1 > 1.0] = 1.0\n ip1[ip1 < -1.0] = -1.0\n ip2[ip2 > 1.0] = 1.0\n ip2[ip2 < -1.0] = -1.0\n beta1 = np.arccos(ip1)\n beta2 = np.arccos(ip2)\n alpha = np.arctan(self._friction_coef)\n antipodal_indices = np.where((beta1 < alpha) & (beta2 < alpha))[0]\n\n # Raise exception if no antipodal pairs.\n num_pairs = antipodal_indices.shape[0]\n if num_pairs == 0:\n return []\n sample_size = min(self._max_rejection_samples, num_pairs)\n grasp_indices = np.random.choice(antipodal_indices,\n size=sample_size,\n replace=False)\n self._logger.debug(\"Grasp comp took %.3f sec\" %\n (time() - pruning_start))\n\n # Compute grasps.\n sample_start = time()\n k = 0\n grasps = []\n while k < sample_size and len(grasps) < num_samples:\n grasp_ind = grasp_indices[k]\n p1 = contact_points1[grasp_ind, :]\n p2 = contact_points2[grasp_ind, :]\n n1 = contact_normals1[grasp_ind, :]\n n2 = contact_normals2[grasp_ind, :]\n # width = np.linalg.norm(p1 - p2)\n k += 1\n\n # Compute center and axis.\n grasp_center = (p1 + p2) // 2\n grasp_axis = p2 - p1\n grasp_axis = grasp_axis / np.linalg.norm(grasp_axis)\n grasp_theta = np.pi / 2\n if grasp_axis[1] != 0:\n grasp_theta = np.arctan2(grasp_axis[0], grasp_axis[1])\n grasp_center_pt = Point(np.array(\n [grasp_center[1], grasp_center[0]]),\n frame=camera_intr.frame)\n\n # Compute grasp points in 3D.\n x1 = point_cloud_im[p1[0], p1[1]]\n x2 = point_cloud_im[p2[0], p2[1]]\n if np.linalg.norm(x2 - x1) > self._gripper_width:\n continue\n\n # Perturb.\n if self._grasp_center_sigma > 0.0:\n grasp_center_pt = grasp_center_pt + ss.multivariate_normal.rvs(\n cov=self._grasp_center_sigma * np.diag(np.ones(2)))\n if self._grasp_angle_sigma > 0.0:\n grasp_theta = grasp_theta + ss.norm.rvs(\n scale=self._grasp_angle_sigma)\n\n # Check center px dist from boundary.\n if (grasp_center[0] < self._min_dist_from_boundary\n or grasp_center[1] < self._min_dist_from_boundary\n or grasp_center[0] >\n depth_im.height - self._min_dist_from_boundary\n or grasp_center[1] >\n depth_im.width - self._min_dist_from_boundary):\n continue\n\n # Sample depths.\n for i in range(self._depth_samples_per_grasp):\n # Get depth in the neighborhood of the center pixel.\n depth_win = depth_im.data[grasp_center[0] -\n self._h:grasp_center[0] + self._h,\n grasp_center[1] -\n self._w:grasp_center[1] + self._w]\n center_depth = np.min(depth_win)\n if center_depth == 0 or np.isnan(center_depth):\n continue\n\n # Sample depth between the min and max.\n min_depth = center_depth + self._min_depth_offset\n max_depth = center_depth + self._max_depth_offset\n sample_depth = min_depth + (max_depth -\n min_depth) * np.random.rand()\n candidate_grasp = Grasp2D(grasp_center_pt,\n grasp_theta,\n sample_depth,\n width=self._gripper_width,\n camera_intr=camera_intr,\n contact_points=[p1, p2],\n contact_normals=[n1, n2])\n\n if visualize:\n vis.figure()\n vis.imshow(depth_im)\n vis.grasp(candidate_grasp)\n vis.scatter(p1[1], p1[0], c=\"b\", s=25)\n vis.scatter(p2[1], p2[0], c=\"b\", s=25)\n vis.show()\n\n grasps.append(candidate_grasp)\n\n # Return sampled grasps.\n self._logger.debug(\"Loop took %.3f sec\" % (time() - sample_start))\n return grasps\n\n\nclass DepthImageSuctionPointSampler(ImageGraspSampler):\n \"\"\"Grasp sampler for suction points from depth images.\n\n Notes\n -----\n Required configuration parameters are specified in Other Parameters.\n\n Other Parameters\n ----------------\n max_suction_dir_optical_axis_angle : float\n Maximum angle, in degrees, between the suction approach axis and the\n camera optical axis.\n delta_theta : float\n Maximum deviation from zero for the aziumth angle of a rotational\n perturbation to the surface normal (for sample diversity).\n delta_phi : float\n Maximum deviation from zero for the elevation angle of a rotational\n perturbation to the surface normal (for sample diversity).\n sigma_depth : float\n Standard deviation for a normal distribution over depth values (for\n sample diversity).\n min_suction_dist : float\n Minimum admissible distance between suction points (for sample\n diversity).\n angle_dist_weight : float\n Amount to weight the angle difference in suction point distance\n computation.\n depth_gaussian_sigma : float\n Sigma used for pre-smoothing the depth image for better gradients.\n \"\"\"\n\n def __init__(self, config):\n # Init superclass.\n ImageGraspSampler.__init__(self, config)\n\n # Read params.\n self._max_suction_dir_optical_axis_angle = np.deg2rad(\n self._config[\"max_suction_dir_optical_axis_angle\"])\n self._max_dist_from_center = self._config[\"max_dist_from_center\"]\n self._min_dist_from_boundary = self._config[\"min_dist_from_boundary\"]\n self._max_num_samples = self._config[\"max_num_samples\"]\n\n self._min_theta = -np.deg2rad(self._config[\"delta_theta\"])\n self._max_theta = np.deg2rad(self._config[\"delta_theta\"])\n self._theta_rv = ss.uniform(loc=self._min_theta,\n scale=self._max_theta - self._min_theta)\n\n self._min_phi = -np.deg2rad(self._config[\"delta_phi\"])\n self._max_phi = np.deg2rad(self._config[\"delta_phi\"])\n self._phi_rv = ss.uniform(loc=self._min_phi,\n scale=self._max_phi - self._min_phi)\n\n self._mean_depth = 0.0\n if \"mean_depth\" in self._config:\n self._mean_depth = self._config[\"mean_depth\"]\n self._sigma_depth = self._config[\"sigma_depth\"]\n self._depth_rv = ss.norm(self._mean_depth, self._sigma_depth**2)\n\n self._min_suction_dist = self._config[\"min_suction_dist\"]\n self._angle_dist_weight = self._config[\"angle_dist_weight\"]\n self._depth_gaussian_sigma = self._config[\"depth_gaussian_sigma\"]\n\n def _sample(self,\n image,\n camera_intr,\n num_samples,\n segmask=None,\n visualize=False,\n constraint_fn=None):\n \"\"\"Sample a set of 2D grasp candidates from a depth image.\n\n Parameters\n ----------\n image : :obj:`autolab_core.RgbdImage` or \"autolab_core.DepthImage\"\n RGB-D or D image to sample from.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the camera that captured the images.\n num_samples : int\n Number of grasps to sample.\n segmask : :obj:`autolab_core.BinaryImage`\n Binary image segmenting out the object of interest.\n visualize : bool\n Whether or not to show intermediate samples (for debugging).\n\n Returns\n -------\n :obj:`list` of :obj:`Grasp2D`\n List of 2D grasp candidates.\n \"\"\"\n if isinstance(image, RgbdImage) or isinstance(image, GdImage):\n depth_im = image.depth\n elif isinstance(image, DepthImage):\n depth_im = image\n else:\n raise ValueError(\n \"image type must be one of [RgbdImage, DepthImage, GdImage]\")\n\n # Sample antipodal pairs in image space.\n grasps = self._sample_suction_points(depth_im,\n camera_intr,\n num_samples,\n segmask=segmask,\n visualize=visualize,\n constraint_fn=constraint_fn)\n return grasps\n\n def _sample_suction_points(self,\n depth_im,\n camera_intr,\n num_samples,\n segmask=None,\n visualize=False,\n constraint_fn=None):\n \"\"\"Sample a set of 2D suction point candidates from a depth image by\n choosing points on an object surface uniformly at random\n and then sampling around the surface normal.\n\n Parameters\n ----------\n depth_im : :obj:\"autolab_core.DepthImage\"\n Depth image to sample from.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the camera that captured the images.\n num_samples : int\n Number of grasps to sample.\n segmask : :obj:`autolab_core.BinaryImage`\n Binary image segmenting out the object of interest.\n visualize : bool\n Whether or not to show intermediate samples (for debugging).\n\n Returns\n -------\n :obj:`list` of :obj:`SuctionPoint2D`\n List of 2D suction point candidates.\n \"\"\"\n # Compute edge pixels.\n filter_start = time()\n if self._depth_gaussian_sigma > 0:\n depth_im_mask = depth_im.apply(snf.gaussian_filter,\n sigma=self._depth_gaussian_sigma)\n else:\n depth_im_mask = depth_im.copy()\n if segmask is not None:\n depth_im_mask = depth_im.mask_binary(segmask)\n self._logger.debug(\"Filtering took %.3f sec\" % (time() - filter_start))\n\n if visualize:\n vis.figure()\n vis.subplot(1, 2, 1)\n vis.imshow(depth_im)\n vis.subplot(1, 2, 2)\n vis.imshow(depth_im_mask)\n vis.show()\n\n # Project to get the point cloud.\n cloud_start = time()\n point_cloud_im = camera_intr.deproject_to_image(depth_im_mask)\n normal_cloud_im = point_cloud_im.normal_cloud_im()\n nonzero_px = depth_im_mask.nonzero_pixels()\n num_nonzero_px = nonzero_px.shape[0]\n if num_nonzero_px == 0:\n return []\n self._logger.debug(\"Normal cloud took %.3f sec\" %\n (time() - cloud_start))\n\n # Randomly sample points and add to image.\n sample_start = time()\n suction_points = []\n k = 0\n sample_size = min(self._max_num_samples, num_nonzero_px)\n indices = np.random.choice(num_nonzero_px,\n size=sample_size,\n replace=False)\n while k < sample_size and len(suction_points) < num_samples:\n # Sample a point uniformly at random.\n ind = indices[k]\n center_px = np.array([nonzero_px[ind, 1], nonzero_px[ind, 0]])\n center = Point(center_px, frame=camera_intr.frame)\n axis = -normal_cloud_im[center.y, center.x]\n depth = point_cloud_im[center.y, center.x][2]\n\n # Update number of tries.\n k += 1\n\n # Check center px dist from boundary.\n if (center_px[0] < self._min_dist_from_boundary\n or center_px[1] < self._min_dist_from_boundary\n or center_px[1] >\n depth_im.height - self._min_dist_from_boundary\n or center_px[0] >\n depth_im.width - self._min_dist_from_boundary):\n continue\n\n # Perturb depth.\n delta_depth = self._depth_rv.rvs(size=1)[0]\n depth = depth + delta_depth\n\n # Keep if the angle between the camera optical axis and the suction\n # direction is less than a threshold.\n dot = max(min(axis.dot(np.array([0, 0, 1])), 1.0), -1.0)\n psi = np.arccos(dot)\n if psi < self._max_suction_dir_optical_axis_angle:\n\n # Create candidate grasp.\n candidate = SuctionPoint2D(center,\n axis,\n depth,\n camera_intr=camera_intr)\n\n # Check constraint satisfaction.\n if constraint_fn is None or constraint_fn(candidate):\n if visualize:\n vis.figure()\n vis.imshow(depth_im)\n vis.scatter(center.x, center.y)\n vis.show()\n\n suction_points.append(candidate)\n self._logger.debug(\"Loop took %.3f sec\" % (time() - sample_start))\n return suction_points\n\n\nclass DepthImageMultiSuctionPointSampler(ImageGraspSampler):\n \"\"\"Grasp sampler for suction points from depth images.\n\n Notes\n -----\n Required configuration parameters are specified in Other Parameters.\n\n Other Parameters\n ----------------\n max_suction_dir_optical_axis_angle : float\n Maximum angle, in degrees, between the suction approach axis and the\n camera optical axis.\n delta_theta : float\n Maximum deviation from zero for the aziumth angle of a rotational\n perturbation to the surface normal (for sample diversity).\n delta_phi : float\n Maximum deviation from zero for the elevation angle of a rotational\n perturbation to the surface normal (for sample diversity).\n sigma_depth : float\n Standard deviation for a normal distribution over depth values (for\n sample diversity).\n min_suction_dist : float\n Minimum admissible distance between suction points (for sample\n diversity).\n angle_dist_weight : float\n Amount to weight the angle difference in suction point distance\n computation.\n depth_gaussian_sigma : float\n Sigma used for pre-smoothing the depth image for better gradients.\n \"\"\"\n\n def __init__(self, config):\n # Init superclass.\n ImageGraspSampler.__init__(self, config)\n\n # Read params.\n self._max_suction_dir_optical_axis_angle = np.deg2rad(\n self._config[\"max_suction_dir_optical_axis_angle\"])\n self._max_dist_from_center = self._config[\"max_dist_from_center\"]\n self._min_dist_from_boundary = self._config[\"min_dist_from_boundary\"]\n self._max_num_samples = self._config[\"max_num_samples\"]\n\n self._min_theta = -np.deg2rad(self._config[\"delta_theta\"])\n self._max_theta = np.deg2rad(self._config[\"delta_theta\"])\n self._theta_rv = ss.uniform(loc=self._min_theta,\n scale=self._max_theta - self._min_theta)\n\n self._min_phi = -np.deg2rad(self._config[\"delta_phi\"])\n self._max_phi = np.deg2rad(self._config[\"delta_phi\"])\n self._phi_rv = ss.uniform(loc=self._min_phi,\n scale=self._max_phi - self._min_phi)\n\n self._mean_depth = 0.0\n if \"mean_depth\" in self._config:\n self._mean_depth = self._config[\"mean_depth\"]\n self._sigma_depth = self._config[\"sigma_depth\"]\n self._depth_rv = ss.norm(self._mean_depth, self._sigma_depth**2)\n\n self._min_suction_dist = self._config[\"min_suction_dist\"]\n self._angle_dist_weight = self._config[\"angle_dist_weight\"]\n self._depth_gaussian_sigma = self._config[\"depth_gaussian_sigma\"]\n\n def _sample(self,\n image,\n camera_intr,\n num_samples,\n segmask=None,\n visualize=False,\n constraint_fn=None):\n \"\"\"Sample a set of 2D grasp candidates from a depth image.\n\n Parameters\n ----------\n image : :obj:`autolab_core.RgbdImage` or `autolab_core.DepthImage`\n RGB-D or D image to sample from.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the camera that captured the images.\n num_samples : int\n Number of grasps to sample.\n segmask : :obj:`autolab_core.BinaryImage`\n Binary image segmenting out the object of interest.\n visualize : bool\n Whether or not to show intermediate samples (for debugging).\n constraint_fn : :obj:`GraspConstraintFn`\n Constraint function to apply to grasps.\n\n Returns\n -------\n :obj:`list` of :obj:`Grasp2D`\n List of 2D grasp candidates.\n \"\"\"\n if isinstance(image, RgbdImage) or isinstance(image, GdImage):\n depth_im = image.depth\n elif isinstance(image, DepthImage):\n depth_im = image\n else:\n raise ValueError(\n \"image type must be one of [RgbdImage, DepthImage, GdImage]\")\n\n # Sample antipodal pairs in image space.\n grasps = self._sample_suction_points(depth_im,\n camera_intr,\n num_samples,\n segmask=segmask,\n visualize=visualize,\n constraint_fn=constraint_fn)\n return grasps\n\n def _sample_suction_points(self,\n depth_im,\n camera_intr,\n num_samples,\n segmask=None,\n visualize=False,\n constraint_fn=None):\n \"\"\"Sample a set of 2D suction point candidates from a depth image by\n choosing points on an object surface uniformly at random\n and then sampling around the surface normal.\n\n Parameters\n ----------\n depth_im : :obj:\"autolab_core.DepthImage\"\n Depth image to sample from.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the camera that captured the images.\n num_samples : int\n Number of grasps to sample.\n segmask : :obj:`autolab_core.BinaryImage`\n Binary image segmenting out the object of interest.\n visualize : bool\n Whether or not to show intermediate samples (for debugging).\n constraint_fn : :obj:`GraspConstraintFn`\n Constraint function to apply to grasps.\n\n Returns\n -------\n :obj:`list` of :obj:`SuctionPoint2D`\n List of 2D suction point candidates.\n \"\"\"\n # Compute edge pixels.\n filter_start = time()\n if self._depth_gaussian_sigma > 0:\n depth_im_mask = depth_im.apply(snf.gaussian_filter,\n sigma=self._depth_gaussian_sigma)\n else:\n depth_im_mask = depth_im.copy()\n if segmask is not None:\n depth_im_mask = depth_im.mask_binary(segmask)\n self._logger.debug(\"Filtering took %.3f sec\" % (time() - filter_start))\n\n if visualize:\n vis.figure()\n vis.subplot(1, 2, 1)\n vis.imshow(depth_im)\n vis.subplot(1, 2, 2)\n vis.imshow(depth_im_mask)\n vis.show()\n\n # Project to get the point cloud.\n cloud_start = time()\n point_cloud_im = camera_intr.deproject_to_image(depth_im_mask)\n normal_cloud_im = point_cloud_im.normal_cloud_im()\n nonzero_px = depth_im_mask.nonzero_pixels()\n num_nonzero_px = nonzero_px.shape[0]\n if num_nonzero_px == 0:\n return []\n self._logger.debug(\"Normal cloud took %.3f sec\" %\n (time() - cloud_start))\n\n # Randomly sample points and add to image.\n sample_start = time()\n suction_points = []\n k = 0\n sample_size = min(self._max_num_samples, num_nonzero_px)\n indices = np.random.choice(num_nonzero_px,\n size=sample_size,\n replace=False)\n while k < sample_size and len(suction_points) < num_samples:\n # Sample a point uniformly at random.\n ind = indices[k]\n center_px = np.array([nonzero_px[ind, 1], nonzero_px[ind, 0]])\n center = Point(center_px, frame=camera_intr.frame)\n axis = -normal_cloud_im[center.y, center.x]\n # depth = point_cloud_im[center.y, center.x][2]\n orientation = 2 * np.pi * np.random.rand()\n\n # Update number of tries.\n k += 1\n\n # Skip bad axes.\n if np.linalg.norm(axis) == 0:\n continue\n\n # Rotation matrix.\n x_axis = axis\n y_axis = np.array([axis[1], -axis[0], 0])\n if np.linalg.norm(y_axis) == 0:\n y_axis = np.array([1, 0, 0])\n y_axis = y_axis / np.linalg.norm(y_axis)\n z_axis = np.cross(x_axis, y_axis)\n R = np.array([x_axis, y_axis, z_axis]).T\n # R_orig = np.copy(R)\n R = R.dot(RigidTransform.x_axis_rotation(orientation))\n t = point_cloud_im[center.y, center.x]\n pose = RigidTransform(rotation=R,\n translation=t,\n from_frame=\"grasp\",\n to_frame=camera_intr.frame)\n\n # Check center px dist from boundary.\n if (center_px[0] < self._min_dist_from_boundary\n or center_px[1] < self._min_dist_from_boundary\n or center_px[1] >\n depth_im.height - self._min_dist_from_boundary\n or center_px[0] >\n depth_im.width - self._min_dist_from_boundary):\n continue\n\n # Keep if the angle between the camera optical axis and the suction\n # direction is less than a threshold.\n dot = max(min(axis.dot(np.array([0, 0, 1])), 1.0), -1.0)\n psi = np.arccos(dot)\n if psi < self._max_suction_dir_optical_axis_angle:\n\n # Check distance to ensure sample diversity.\n candidate = MultiSuctionPoint2D(pose, camera_intr=camera_intr)\n\n # Check constraint satisfaction.\n if constraint_fn is None or constraint_fn(candidate):\n if visualize:\n vis.figure()\n vis.imshow(depth_im)\n vis.scatter(center.x, center.y)\n vis.show()\n\n suction_points.append(candidate)\n self._logger.debug(\"Loop took %.3f sec\" % (time() - sample_start))\n return suction_points\n\n\nclass ImageGraspSamplerFactory(object):\n \"\"\"Factory for image grasp samplers.\"\"\"\n\n @staticmethod\n def sampler(sampler_type, config):\n if sampler_type == \"antipodal_depth\":\n return AntipodalDepthImageGraspSampler(config)\n elif sampler_type == \"suction\":\n return DepthImageSuctionPointSampler(config)\n elif sampler_type == \"multi_suction\":\n return DepthImageMultiSuctionPointSampler(config)\n else:\n raise ValueError(\"Image grasp sampler type %s not supported!\" %\n (sampler_type))\n" }, { "path": "gqcnn/grasping/policy/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\"\"\"\nfrom .fc_policy import (FullyConvolutionalGraspingPolicyParallelJaw,\n FullyConvolutionalGraspingPolicySuction)\nfrom .policy import (RobustGraspingPolicy, CrossEntropyRobustGraspingPolicy,\n RgbdImageState, GraspAction, UniformRandomGraspingPolicy)\n\n__all__ = [\n \"FullyConvolutionalGraspingPolicyParallelJaw\",\n \"FullyConvolutionalGraspingPolicySuction\", \"RobustGraspingPolicy\",\n \"CrossEntropyRobustGraspingPolicy\", \"UniformRandomGraspingPolicy\",\n \"RgbdImageState\", \"GraspAction\"\n]\n" }, { "path": "gqcnn/grasping/policy/enums.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nEnums for GQ-CNN policies.\n\nAuthor\n------\nVishal Satish\n\"\"\"\n\n\nclass SamplingMethod(object):\n TOP_K = \"top_k\"\n UNIFORM = \"uniform\"\n" }, { "path": "gqcnn/grasping/policy/fc_policy.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nFully-Convolutional GQ-CNN grasping policies.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nfrom abc import abstractmethod\nimport os\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nfrom autolab_core import Point, Logger, DepthImage\nfrom visualization import Visualizer2D as vis\n\nfrom ...grasping import Grasp2D, SuctionPoint2D\nfrom ...utils import GeneralConstants, NoValidGraspsException\n\nfrom .enums import SamplingMethod\nfrom .policy import GraspingPolicy, GraspAction\n\n\nclass FullyConvolutionalGraspingPolicy(GraspingPolicy):\n \"\"\"Abstract grasp sampling policy class using Fully-Convolutional GQ-CNN\n network.\"\"\"\n\n def __init__(self, cfg, filters=None):\n \"\"\"\n Parameters\n ----------\n cfg : dict\n Python dictionary of policy configuration parameters.\n filters : dict\n Python dictionary of kinematic filters to apply.\n \"\"\"\n GraspingPolicy.__init__(self, cfg, init_sampler=False)\n\n # Init logger.\n self._logger = Logger.get_logger(self.__class__.__name__)\n\n self._cfg = cfg\n self._sampling_method = self._cfg[\"sampling_method\"]\n\n # GQ-CNN parameters.\n self._gqcnn_stride = self._cfg[\"gqcnn_stride\"]\n self._gqcnn_recep_h = self._cfg[\"gqcnn_recep_h\"]\n self._gqcnn_recep_w = self._cfg[\"gqcnn_recep_w\"]\n\n # Grasp filtering.\n self._filters = filters\n self._max_grasps_to_filter = self._cfg[\"max_grasps_to_filter\"]\n self._filter_grasps = self._cfg[\"filter_grasps\"]\n\n # Visualization parameters.\n self._vis_config = self._cfg[\"policy_vis\"]\n self._vis_scale = self._vis_config[\"scale\"]\n self._vis_show_axis = self._vis_config[\"show_axis\"]\n\n self._num_vis_samples = self._vis_config[\"num_samples\"]\n self._vis_actions_2d = self._vis_config[\"actions_2d\"]\n self._vis_actions_3d = self._vis_config[\"actions_3d\"]\n\n self._vis_affordance_map = self._vis_config[\"affordance_map\"]\n\n self._vis_output_dir = None\n # If this exists in the config then all visualizations will be logged\n # here instead of displayed.\n if \"output_dir\" in self._vis_config:\n self._vis_output_dir = self._vis_config[\"output_dir\"]\n self._state_counter = 0\n\n def _unpack_state(self, state):\n \"\"\"Unpack information from the provided `RgbdImageState`.\"\"\"\n # TODO(vsatish): Don't access raw depth data like this.\n return (state.rgbd_im.depth, state.rgbd_im.depth._data,\n state.segmask.raw_data, state.camera_intr)\n\n def _mask_predictions(self, preds, raw_segmask):\n \"\"\"Mask the given predictions with the given segmask, setting the rest\n to 0.0.\"\"\"\n preds_masked = np.zeros_like(preds)\n raw_segmask_cropped = raw_segmask[self._gqcnn_recep_h //\n 2:raw_segmask.shape[0] -\n self._gqcnn_recep_h // 2,\n self._gqcnn_recep_w //\n 2:raw_segmask.shape[1] -\n self._gqcnn_recep_w // 2, 0]\n raw_segmask_downsampled = raw_segmask_cropped[::self._gqcnn_stride, ::\n self._gqcnn_stride]\n if raw_segmask_downsampled.shape[0] != preds.shape[1]:\n raw_segmask_downsampled_new = np.zeros(preds.shape[1:3])\n raw_segmask_downsampled_new[:raw_segmask_downsampled.\n shape[0], :raw_segmask_downsampled.\n shape[1]] = raw_segmask_downsampled\n raw_segmask_downsampled = raw_segmask_downsampled_new\n nonzero_mask_ind = np.where(raw_segmask_downsampled > 0)\n preds_masked[:, nonzero_mask_ind[0],\n nonzero_mask_ind[1]] = preds[:, nonzero_mask_ind[0],\n nonzero_mask_ind[1]]\n return preds_masked\n\n def _sample_predictions(self, preds, num_actions):\n \"\"\"Sample predictions.\"\"\"\n dim2 = preds.shape[2]\n dim1 = preds.shape[1]\n dim3 = preds.shape[3]\n preds_flat = np.ravel(preds)\n pred_ind_flat = self._sample_predictions_flat(preds_flat, num_actions)\n pred_ind = np.zeros((num_actions, len(preds.shape)), dtype=np.int32)\n for idx in range(num_actions):\n pred_ind[idx, 0] = pred_ind_flat[idx] // (dim2 * dim1 * dim3)\n pred_ind[idx, 1] = (pred_ind_flat[idx] -\n (pred_ind[idx, 0] *\n (dim2 * dim1 * dim3))) // (dim2 * dim3)\n pred_ind[idx, 2] = (pred_ind_flat[idx] - (pred_ind[idx, 0] *\n (dim2 * dim1 * dim3)) -\n (pred_ind[idx, 1] * (dim2 * dim3))) // dim3\n pred_ind[idx, 3] = (pred_ind_flat[idx] - (pred_ind[idx, 0] *\n (dim2 * dim1 * dim3)) -\n (pred_ind[idx, 1] * (dim2 * dim3))) % dim3\n return pred_ind\n\n def _sample_predictions_flat(self, preds_flat, num_samples):\n \"\"\"Helper function to do the actual sampling.\"\"\"\n if num_samples == 1:\n # `argmax` is faster than `argpartition` for special case of single\n # sample.\n if self._sampling_method == SamplingMethod.TOP_K:\n return [np.argmax(preds_flat)]\n elif self._sampling_method == SamplingMethod.UNIFORM:\n nonzero_ind = np.where(preds_flat > 0)[0]\n return np.random.choice(nonzero_ind)\n else:\n raise ValueError(\"Invalid sampling method: {}\".format(\n self._sampling_method))\n else:\n if self._sampling_method == \"top_k\":\n return np.argpartition(preds_flat,\n -1 * num_samples)[-1 * num_samples:]\n elif self._sampling_method == \"uniform\":\n nonzero_ind = np.where(preds_flat > 0)[0]\n if nonzero_ind.shape[0] == 0:\n raise NoValidGraspsException(\n \"No grasps with nonzero quality\")\n return np.random.choice(nonzero_ind, size=num_samples)\n else:\n raise ValueError(\"Invalid sampling method: {}\".format(\n self._sampling_method))\n\n @abstractmethod\n def _get_actions(self, preds, ind, images, depths, camera_intr,\n num_actions):\n \"\"\"Generate the actions to be returned.\"\"\"\n pass\n\n @abstractmethod\n def _visualize_3d(self, actions, wrapped_depth_im, camera_intr,\n num_actions):\n \"\"\"Visualize the actions in 3D.\"\"\"\n pass\n\n @abstractmethod\n def _visualize_affordance_map(self,\n preds,\n depth_im,\n scale,\n plot_max=True,\n output_dir=None):\n \"\"\"Visualize an affordance map of the network predictions overlayed on\n the depth image.\"\"\"\n pass\n\n def _visualize_2d(self,\n actions,\n preds,\n wrapped_depth_im,\n num_actions,\n scale,\n show_axis,\n output_dir=None):\n \"\"\"Visualize the actions in 2D.\"\"\"\n self._logger.info(\"Visualizing actions in 2d...\")\n\n # Plot actions in 2D.\n vis.figure()\n vis.imshow(wrapped_depth_im)\n for i in range(num_actions):\n vis.grasp(actions[i].grasp,\n scale=scale,\n show_axis=show_axis,\n color=plt.cm.RdYlGn(actions[i].q_value))\n vis.title(\"Top {} Grasps\".format(num_actions))\n if output_dir is not None:\n vis.savefig(os.path.join(output_dir, \"top_grasps.png\"))\n else:\n vis.show()\n\n def _filter(self, actions):\n \"\"\"Filter actions.\"\"\"\n for action in actions:\n valid = True\n for filter_name, is_valid in self._filters.items():\n if not is_valid(action.grasp):\n self._logger.info(\n \"Grasp {} is not valid with filter {}\".format(\n action.grasp, filter_name))\n valid = False\n break\n if valid:\n return action\n raise NoValidGraspsException(\"No grasps found after filtering!\")\n\n @abstractmethod\n def _gen_images_and_depths(self, depth, segmask):\n \"\"\"Generate inputs for the grasp quality function.\"\"\"\n pass\n\n def _action(self, state, num_actions=1):\n \"\"\"Plan action(s).\"\"\"\n if self._filter_grasps:\n assert self._filters is not None, (\"Trying to filter grasps but no\"\n \" filters were provided!\")\n assert num_actions == 1, (\"Filtering support is only implemented\"\n \" for single actions!\")\n num_actions = self._max_grasps_to_filter\n\n # Set up log dir for state visualizations.\n state_output_dir = None\n if self._vis_output_dir is not None:\n state_output_dir = os.path.join(\n self._vis_output_dir,\n \"state_{}\".format(str(self._state_counter).zfill(5)))\n if not os.path.exists(state_output_dir):\n os.makedirs(state_output_dir)\n self._state_counter += 1\n\n # Unpack the `RgbdImageState`.\n wrapped_depth, raw_depth, raw_seg, camera_intr = self._unpack_state(\n state)\n\n # Predict.\n images, depths = self._gen_images_and_depths(raw_depth, raw_seg)\n preds = self._grasp_quality_fn.quality(images, depths)\n\n # Get success probablility predictions only (this is needed because the\n # output of the network is pairs of (p_failure, p_success)).\n preds_success_only = preds[:, :, :, 1::2]\n\n # Mask predicted success probabilities with the cropped and downsampled\n # object segmask so we only sample grasps on the objects.\n preds_success_only = self._mask_predictions(preds_success_only,\n raw_seg)\n\n # If we want to visualize more than one action, we have to sample more.\n # TODO(vsatish): If this is used with the \"top_k\" sampling method, the\n # final returned action is not the best because the argpartition does\n # not sort the partitioned indices. Fix this.\n num_actions_to_sample = self._num_vis_samples if (\n self._vis_actions_2d or self._vis_actions_3d) else num_actions\n\n if (self._sampling_method == SamplingMethod.TOP_K\n and self._num_vis_samples):\n self._logger.warning(\"FINAL GRASP RETURNED IS NOT THE BEST!\")\n\n # Sample num_actions_to_sample indices from the success predictions.\n sampled_ind = self._sample_predictions(preds_success_only,\n num_actions_to_sample)\n\n # Wrap actions to be returned.\n actions = self._get_actions(preds_success_only, sampled_ind, images,\n depths, camera_intr, num_actions_to_sample)\n\n # Filter grasps.\n if self._filter_grasps:\n actions = sorted(actions,\n reverse=True,\n key=lambda action: action.q_value)\n actions = [self._filter(actions)]\n\n # Visualize.\n if self._vis_actions_3d:\n self._logger.info(\"Generating 3D Visualization...\")\n self._visualize_3d(actions, wrapped_depth, camera_intr,\n num_actions_to_sample)\n if self._vis_actions_2d:\n self._logger.info(\"Generating 2D visualization...\")\n self._visualize_2d(actions,\n preds_success_only,\n wrapped_depth,\n num_actions_to_sample,\n self._vis_scale,\n self._vis_show_axis,\n output_dir=state_output_dir)\n if self._vis_affordance_map:\n self._visualize_affordance_map(preds_success_only,\n wrapped_depth,\n self._vis_scale,\n output_dir=state_output_dir)\n\n return actions[-1] if (self._filter_grasps or num_actions\n == 1) else actions[-(num_actions + 1):]\n\n def action_set(self, state, num_actions):\n \"\"\"Plan a set of actions.\n\n Parameters\n ----------\n state : :obj:`gqcnn.RgbdImageState`\n The RGBD image state.\n num_actions : int\n The number of actions to plan.\n\n Returns\n ------\n list of :obj:`gqcnn.GraspAction`\n The planned grasps.\n \"\"\"\n return [\n action.grasp\n for action in self._action(state, num_actions=num_actions)\n ]\n\n\nclass FullyConvolutionalGraspingPolicyParallelJaw(\n FullyConvolutionalGraspingPolicy):\n \"\"\"Parallel jaw grasp sampling policy using the FC-GQ-CNN.\"\"\"\n\n def __init__(self, cfg, filters=None):\n \"\"\"\n Parameters\n ----------\n cfg : dict\n Python dictionary of policy configuration parameters.\n filters : dict\n Python dictionary of functions to apply to filter invalid grasps.\n \"\"\"\n FullyConvolutionalGraspingPolicy.__init__(self, cfg, filters=filters)\n\n self._gripper_width = self._cfg[\"gripper_width\"]\n\n # Depth sampling parameters.\n self._num_depth_bins = self._cfg[\"num_depth_bins\"]\n self._depth_offset = 0.0 # Hard offset from the workspace.\n if \"depth_offset\" in self._cfg:\n self._depth_offset = self._cfg[\"depth_offset\"]\n\n def _sample_depths(self, raw_depth_im, raw_seg):\n \"\"\"Sample depths from the raw depth image.\"\"\"\n max_depth = np.max(raw_depth_im) + self._depth_offset\n\n # For sampling the min depth, we only sample from the portion of the\n # depth image in the object segmask because sometimes the rim of the\n # bin is not properly subtracted out of the depth image.\n raw_depth_im_segmented = np.ones_like(raw_depth_im)\n raw_depth_im_segmented[np.where(raw_seg > 0)] = raw_depth_im[np.where(\n raw_seg > 0)]\n min_depth = np.min(raw_depth_im_segmented) + self._depth_offset\n\n depth_bin_width = (max_depth - min_depth) / self._num_depth_bins\n depths = np.zeros((self._num_depth_bins, 1))\n for i in range(self._num_depth_bins):\n depths[i][0] = min_depth + (i * depth_bin_width +\n depth_bin_width / 2)\n return depths\n\n def _get_actions(self, preds, ind, images, depths, camera_intr,\n num_actions):\n \"\"\"Generate the actions to be returned.\"\"\"\n actions = []\n # TODO(vsatish): These should use the max angle instead.\n ang_bin_width = GeneralConstants.PI / preds.shape[-1]\n for i in range(num_actions):\n im_idx = ind[i, 0]\n h_idx = ind[i, 1]\n w_idx = ind[i, 2]\n ang_idx = ind[i, 3]\n center = Point(\n np.asarray([\n w_idx * self._gqcnn_stride + self._gqcnn_recep_w // 2,\n h_idx * self._gqcnn_stride + self._gqcnn_recep_h // 2\n ]))\n ang = GeneralConstants.PI / 2 - (ang_idx * ang_bin_width +\n ang_bin_width / 2)\n depth = depths[im_idx, 0]\n grasp = Grasp2D(center,\n ang,\n depth,\n width=self._gripper_width,\n camera_intr=camera_intr)\n grasp_action = GraspAction(grasp, preds[im_idx, h_idx, w_idx,\n ang_idx],\n DepthImage(images[im_idx]))\n actions.append(grasp_action)\n return actions\n\n def _gen_images_and_depths(self, depth, segmask):\n \"\"\"Replicate the depth image and sample corresponding depths.\"\"\"\n depths = self._sample_depths(depth, segmask)\n images = np.tile(np.asarray([depth]), (self._num_depth_bins, 1, 1, 1))\n return images, depths\n\n def _visualize_3d(self, actions, wrapped_depth_im, camera_intr,\n num_actions):\n \"\"\"Visualize the actions in 3D.\"\"\"\n raise NotImplementedError\n\n def _visualize_affordance_map(self, preds, depth_im):\n \"\"\"Visualize an affordance map of the network predictions overlayed on\n the depth image.\"\"\"\n raise NotImplementedError\n\n\nclass FullyConvolutionalGraspingPolicySuction(FullyConvolutionalGraspingPolicy\n ):\n \"\"\"Suction grasp sampling policy using the FC-GQ-CNN.\"\"\"\n\n def _get_actions(self, preds, ind, images, depths, camera_intr,\n num_actions):\n \"\"\"Generate the actions to be returned.\"\"\"\n depth_im = DepthImage(images[0], frame=camera_intr.frame)\n point_cloud_im = camera_intr.deproject_to_image(depth_im)\n normal_cloud_im = point_cloud_im.normal_cloud_im()\n\n actions = []\n for i in range(num_actions):\n im_idx = ind[i, 0]\n h_idx = ind[i, 1]\n w_idx = ind[i, 2]\n center = Point(\n np.asarray([\n w_idx * self._gqcnn_stride + self._gqcnn_recep_w // 2,\n h_idx * self._gqcnn_stride + self._gqcnn_recep_h // 2\n ]))\n axis = -normal_cloud_im[center.y, center.x]\n if np.linalg.norm(axis) == 0:\n continue\n depth = depth_im[center.y, center.x, 0]\n if depth == 0.0:\n continue\n grasp = SuctionPoint2D(center,\n axis=axis,\n depth=depth,\n camera_intr=camera_intr)\n grasp_action = GraspAction(grasp, preds[im_idx, h_idx, w_idx, 0],\n DepthImage(images[im_idx]))\n actions.append(grasp_action)\n return actions\n\n def _visualize_affordance_map(self,\n preds,\n depth_im,\n scale,\n plot_max=True,\n output_dir=None):\n \"\"\"Visualize an affordance map of the network predictions overlayed on\n the depth image.\"\"\"\n self._logger.info(\"Visualizing affordance map...\")\n\n affordance_map = preds[0, ..., 0]\n tf_depth_im = depth_im.crop(\n depth_im.shape[0] - self._gqcnn_recep_h, depth_im.shape[1] -\n self._gqcnn_recep_w).resize(1.0 / self._gqcnn_stride)\n\n # Plot.\n vis.figure()\n vis.imshow(tf_depth_im)\n plt.imshow(affordance_map,\n cmap=plt.cm.RdYlGn,\n alpha=0.3,\n vmin=0.0,\n vmax=1.0)\n if plot_max:\n affordance_argmax = np.unravel_index(np.argmax(affordance_map),\n affordance_map.shape)\n plt.scatter(affordance_argmax[1],\n affordance_argmax[0],\n c=\"black\",\n marker=\".\",\n s=scale * 25)\n vis.title(\"Grasp Affordance Map\")\n if output_dir is not None:\n vis.savefig(os.path.join(output_dir, \"grasp_affordance_map.png\"))\n else:\n vis.show()\n\n def _gen_images_and_depths(self, depth, segmask):\n \"\"\"Extend the image to a 4D tensor.\"\"\"\n # TODO(vsatish): Depth should be made an optional input to the network.\n return np.expand_dims(depth, 0), np.array([-1])\n\n def _visualize_3d(self, actions, wrapped_depth_im, camera_intr,\n num_actions):\n \"\"\"Visualize the actions in 3D.\"\"\"\n raise NotImplementedError\n" }, { "path": "gqcnn/grasping/policy/policy.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nGrasping policies.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nfrom abc import ABC, abstractmethod\nimport copy\nimport pickle as pkl\nimport math\nimport os\nfrom time import time\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport scipy.ndimage.filters as snf\nimport scipy.stats as ss\nfrom sklearn.mixture import GaussianMixture\n\nimport autolab_core.utils as utils\nfrom autolab_core import (Point, Logger, BinaryImage, ColorImage,\n DepthImage, RgbdImage, SegmentationImage,\n CameraIntrinsics)\nfrom visualization import Visualizer2D as vis\n\nfrom ..constraint_fn import GraspConstraintFnFactory\nfrom ..grasp import Grasp2D, SuctionPoint2D, MultiSuctionPoint2D\nfrom ..grasp_quality_function import (GraspQualityFunctionFactory,\n GQCnnQualityFunction)\nfrom ..image_grasp_sampler import ImageGraspSamplerFactory\nfrom ...utils import GeneralConstants, NoValidGraspsException\n\n\nclass RgbdImageState(object):\n \"\"\"State to encapsulate RGB-D images.\"\"\"\n\n def __init__(self,\n rgbd_im,\n camera_intr,\n segmask=None,\n obj_segmask=None,\n fully_observed=None):\n \"\"\"\n Parameters\n ----------\n rgbd_im : :obj:`autolab_core.RgbdImage`\n An RGB-D image to plan grasps on.\n camera_intr : :obj:`autolab_core.CameraIntrinsics`\n Intrinsics of the RGB-D camera.\n segmask : :obj:`autolab_core.BinaryImage`\n Segmentation mask for the image.\n obj_segmask : :obj:`autolab_core.SegmentationImage`\n Segmentation mask for the different objects in the image.\n full_observed : :obj:`object`\n Representation of the fully observed state.\n \"\"\"\n self.rgbd_im = rgbd_im\n self.camera_intr = camera_intr\n self.segmask = segmask\n self.obj_segmask = obj_segmask\n self.fully_observed = fully_observed\n\n def save(self, save_dir):\n \"\"\"Save to a directory.\n\n Parameters\n ----------\n save_dir : str\n The directory to save to.\n \"\"\"\n if not os.path.exists(save_dir):\n os.mkdir(save_dir)\n color_image_filename = os.path.join(save_dir, \"color.png\")\n depth_image_filename = os.path.join(save_dir, \"depth.npy\")\n camera_intr_filename = os.path.join(save_dir, \"camera.intr\")\n segmask_filename = os.path.join(save_dir, \"segmask.npy\")\n obj_segmask_filename = os.path.join(save_dir, \"obj_segmask.npy\")\n state_filename = os.path.join(save_dir, \"state.pkl\")\n self.rgbd_im.color.save(color_image_filename)\n self.rgbd_im.depth.save(depth_image_filename)\n self.camera_intr.save(camera_intr_filename)\n if self.segmask is not None:\n self.segmask.save(segmask_filename)\n if self.obj_segmask is not None:\n self.obj_segmask.save(obj_segmask_filename)\n if self.fully_observed is not None:\n pkl.dump(self.fully_observed, open(state_filename, \"wb\"))\n\n @staticmethod\n def load(save_dir):\n \"\"\"Load an :obj:`RGBDImageState`.\n\n Parameters\n ----------\n save_dir : str\n The directory to load from.\n \"\"\"\n if not os.path.exists(save_dir):\n raise ValueError(\"Directory %s does not exist!\" % (save_dir))\n color_image_filename = os.path.join(save_dir, \"color.png\")\n depth_image_filename = os.path.join(save_dir, \"depth.npy\")\n camera_intr_filename = os.path.join(save_dir, \"camera.intr\")\n segmask_filename = os.path.join(save_dir, \"segmask.npy\")\n obj_segmask_filename = os.path.join(save_dir, \"obj_segmask.npy\")\n state_filename = os.path.join(save_dir, \"state.pkl\")\n camera_intr = CameraIntrinsics.load(camera_intr_filename)\n color = ColorImage.open(color_image_filename, frame=camera_intr.frame)\n depth = DepthImage.open(depth_image_filename, frame=camera_intr.frame)\n segmask = None\n if os.path.exists(segmask_filename):\n segmask = BinaryImage.open(segmask_filename,\n frame=camera_intr.frame)\n obj_segmask = None\n if os.path.exists(obj_segmask_filename):\n obj_segmask = SegmentationImage.open(obj_segmask_filename,\n frame=camera_intr.frame)\n fully_observed = None\n if os.path.exists(state_filename):\n fully_observed = pkl.load(open(state_filename, \"rb\"))\n return RgbdImageState(RgbdImage.from_color_and_depth(color, depth),\n camera_intr,\n segmask=segmask,\n obj_segmask=obj_segmask,\n fully_observed=fully_observed)\n\n\nclass GraspAction(object):\n \"\"\"Action to encapsulate grasps.\n \"\"\"\n\n def __init__(self, grasp, q_value, image=None, policy_name=None):\n \"\"\"\n Parameters\n ----------\n grasp : :obj`Grasp2D` or :obj:`SuctionPoint2D`\n 2D grasp to wrap.\n q_value : float\n Grasp quality.\n image : :obj:`autolab_core.DepthImage`\n Depth image corresponding to grasp.\n policy_name : str\n Policy name.\n \"\"\"\n self.grasp = grasp\n self.q_value = q_value\n self.image = image\n self.policy_name = policy_name\n\n def save(self, save_dir):\n \"\"\"Save grasp action.\n\n Parameters\n ----------\n save_dir : str\n Directory to save the grasp action to.\n \"\"\"\n if not os.path.exists(save_dir):\n os.mkdir(save_dir)\n grasp_filename = os.path.join(save_dir, \"grasp.pkl\")\n q_value_filename = os.path.join(save_dir, \"pred_robustness.pkl\")\n image_filename = os.path.join(save_dir, \"tf_image.npy\")\n pkl.dump(self.grasp, open(grasp_filename, \"wb\"))\n pkl.dump(self.q_value, open(q_value_filename, \"wb\"))\n if self.image is not None:\n self.image.save(image_filename)\n\n @staticmethod\n def load(save_dir):\n \"\"\"Load a saved grasp action.\n\n Parameters\n ----------\n save_dir : str\n Directory of the saved grasp action.\n\n Returns\n -------\n :obj:`GraspAction`\n Loaded grasp action.\n \"\"\"\n if not os.path.exists(save_dir):\n raise ValueError(\"Directory %s does not exist!\" % (save_dir))\n grasp_filename = os.path.join(save_dir, \"grasp.pkl\")\n q_value_filename = os.path.join(save_dir, \"pred_robustness.pkl\")\n image_filename = os.path.join(save_dir, \"tf_image.npy\")\n grasp = pkl.load(open(grasp_filename, \"rb\"))\n q_value = pkl.load(open(q_value_filename, \"rb\"))\n image = None\n if os.path.exists(image_filename):\n image = DepthImage.open(image_filename)\n return GraspAction(grasp, q_value, image)\n\n\nclass Policy(ABC):\n \"\"\"Abstract policy class.\"\"\"\n\n def __call__(self, state):\n \"\"\"Execute the policy on a state.\"\"\"\n return self.action(state)\n\n @abstractmethod\n def action(self, state):\n \"\"\"Returns an action for a given state.\"\"\"\n pass\n\n\nclass GraspingPolicy(Policy):\n \"\"\"Policy for robust grasping with Grasp Quality Convolutional Neural\n Networks (GQ-CNN).\"\"\"\n\n def __init__(self, config, init_sampler=True):\n \"\"\"\n Parameters\n ----------\n config : dict\n Python dictionary of parameters for the policy.\n init_sampler : bool\n Whether or not to initialize the grasp sampler.\n\n Notes\n -----\n Required configuration parameters are specified in Other Parameters.\n\n Other Parameters\n ----------------\n sampling : dict\n Dictionary of parameters for grasp sampling, see\n `gqcnn/grasping/image_grasp_sampler.py`.\n gqcnn_model : str\n String path to a trained GQ-CNN model.\n \"\"\"\n # Store parameters.\n self._config = config\n self._gripper_width = 0.05\n if \"gripper_width\" in config:\n self._gripper_width = config[\"gripper_width\"]\n\n # Set the logging dir and possibly log file.\n self._logging_dir = None\n log_file = None\n if \"logging_dir\" in self.config:\n self._logging_dir = self.config[\"logging_dir\"]\n if not os.path.exists(self._logging_dir):\n os.makedirs(self._logging_dir)\n log_file = os.path.join(self._logging_dir, \"policy.log\")\n\n # Setup logger.\n self._logger = Logger.get_logger(self.__class__.__name__,\n log_file=log_file,\n global_log_file=True)\n\n # Init grasp sampler.\n if init_sampler:\n self._sampling_config = config[\"sampling\"]\n self._sampling_config[\"gripper_width\"] = self._gripper_width\n if \"crop_width\" in config[\"metric\"] and \"crop_height\" in config[\n \"metric\"]:\n pad = max(\n math.ceil(\n np.sqrt(2) * (config[\"metric\"][\"crop_width\"] / 2)),\n math.ceil(\n np.sqrt(2) * (config[\"metric\"][\"crop_height\"] / 2)))\n self._sampling_config[\"min_dist_from_boundary\"] = pad\n self._sampling_config[\"gripper_width\"] = self._gripper_width\n sampler_type = self._sampling_config[\"type\"]\n self._grasp_sampler = ImageGraspSamplerFactory.sampler(\n sampler_type, self._sampling_config)\n\n # Init constraint function.\n self._grasp_constraint_fn = None\n if \"constraints\" in self._config:\n self._constraint_config = self._config[\"constraints\"]\n constraint_type = self._constraint_config[\"type\"]\n self._grasp_constraint_fn = GraspConstraintFnFactory.constraint_fn(\n constraint_type, self._constraint_config)\n\n # Init grasp quality function.\n self._metric_config = config[\"metric\"]\n metric_type = self._metric_config[\"type\"]\n self._grasp_quality_fn = GraspQualityFunctionFactory.quality_function(\n metric_type, self._metric_config)\n\n @property\n def config(self):\n \"\"\"Returns the policy configuration parameters.\n\n Returns\n -------\n dict\n Python dictionary of the policy configuration parameters.\n \"\"\"\n return self._config\n\n @property\n def grasp_sampler(self):\n \"\"\"Returns the grasp sampler.\n\n Returns\n -------\n :obj:`gqcnn.grasping.image_grasp_sampler.ImageGraspSampler`\n The grasp sampler.\n \"\"\"\n return self._grasp_sampler\n\n @property\n def grasp_quality_fn(self):\n \"\"\"Returns the grasp quality function.\n\n Returns\n -------\n :obj:`gqcnn.grasping.grasp_quality_function.GraspQualityFunction`\n The grasp quality function.\n \"\"\"\n return self._grasp_quality_fn\n\n @property\n def grasp_constraint_fn(self):\n \"\"\"Returns the grasp constraint function.\n\n Returns\n -------\n :obj:`gqcnn.grasping.constraint_fn.GraspConstraintFn`\n The grasp contraint function.\n \"\"\"\n return self._grasp_constraint_fn\n\n @property\n def gqcnn(self):\n \"\"\"Returns the GQ-CNN.\n\n Returns\n -------\n :obj:`gqcnn.model.tf.GQCNNTF`\n The GQ-CNN model.\n \"\"\"\n return self._gqcnn\n\n def set_constraint_fn(self, constraint_fn):\n \"\"\"Sets the grasp constraint function.\n\n Parameters\n ----------\n constraint_fn : :obj`gqcnn.grasping.constraint_fn.GraspConstraintFn`\n The grasp contraint function.\n \"\"\"\n self._grasp_constraint_fn = constraint_fn\n\n def action(self, state):\n \"\"\"Returns an action for a given state.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n The RGB-D image state to plan grasps on.\n\n Returns\n -------\n :obj:`GraspAction`\n The planned grasp action.\n \"\"\"\n # Save state.\n if self._logging_dir is not None:\n policy_id = utils.gen_experiment_id()\n policy_dir = os.path.join(self._logging_dir,\n \"policy_output_%s\" % (policy_id))\n while os.path.exists(policy_dir):\n policy_id = utils.gen_experiment_id()\n policy_dir = os.path.join(self._logging_dir,\n \"policy_output_%s\" % (policy_id))\n self._policy_dir = policy_dir\n os.mkdir(self._policy_dir)\n state_dir = os.path.join(self._policy_dir, \"state\")\n state.save(state_dir)\n\n # Plan action.\n action = self._action(state)\n\n # Save action.\n if self._logging_dir is not None:\n action_dir = os.path.join(self._policy_dir, \"action\")\n action.save(action_dir)\n return action\n\n @abstractmethod\n def _action(self, state):\n \"\"\"Returns an action for a given state.\n \"\"\"\n pass\n\n def show(self, filename=None, dpi=100):\n \"\"\"Show a figure.\n\n Parameters\n ----------\n filename : str\n File to save figure to.\n dpi : int\n Dpi of figure.\n \"\"\"\n if self._logging_dir is None:\n vis.show()\n else:\n filename = os.path.join(self._policy_dir, filename)\n vis.savefig(filename, dpi=dpi)\n\n\nclass UniformRandomGraspingPolicy(GraspingPolicy):\n \"\"\"Returns a grasp uniformly at random.\"\"\"\n\n def __init__(self, config):\n \"\"\"\n Parameters\n ----------\n config : dict\n Python dictionary of policy configuration parameters.\n filters : dict\n Python dictionary of functions to apply to filter invalid grasps.\n \"\"\"\n GraspingPolicy.__init__(self, config)\n self._num_grasp_samples = 1\n\n self._grasp_center_std = 0.0\n if \"grasp_center_std\" in config:\n self._grasp_center_std = config[\"grasp_center_std\"]\n\n def _action(self, state):\n \"\"\"Plans the grasp with the highest probability of success on\n the given RGB-D image.\n\n Attributes\n ----------\n state : :obj:`RgbdImageState`\n Image to plan grasps on.\n\n Returns\n -------\n :obj:`GraspAction`\n Grasp to execute.\n \"\"\"\n # Check valid input.\n if not isinstance(state, RgbdImageState):\n raise ValueError(\"Must provide an RGB-D image state.\")\n\n # Parse state.\n rgbd_im = state.rgbd_im\n camera_intr = state.camera_intr\n segmask = state.segmask\n\n # Sample grasps.\n grasps = self._grasp_sampler.sample(\n rgbd_im,\n camera_intr,\n self._num_grasp_samples,\n segmask=segmask,\n visualize=self.config[\"vis\"][\"grasp_sampling\"],\n constraint_fn=self._grasp_constraint_fn,\n seed=None)\n num_grasps = len(grasps)\n if num_grasps == 0:\n self._logger.warning(\"No valid grasps could be found\")\n raise NoValidGraspsException()\n\n # Set grasp.\n grasp = grasps[0]\n\n # Perturb grasp.\n if self._grasp_center_std > 0.0:\n grasp_center_rv = ss.multivariate_normal(\n grasp.center.data, cov=self._grasp_center_std**2)\n grasp.center.data = grasp_center_rv.rvs(size=1)[0]\n\n # Form tensors.\n return GraspAction(grasp, 0.0, state.rgbd_im.depth)\n\n\nclass RobustGraspingPolicy(GraspingPolicy):\n \"\"\"Samples a set of grasp candidates in image space,\n ranks the grasps by the predicted probability of success from a GQ-CNN\n and returns the grasp with the highest probability of success.\"\"\"\n\n def __init__(self, config, filters=None):\n \"\"\"\n Parameters\n ----------\n config : dict\n Python dictionary of policy configuration parameters.\n filters : dict\n Python dictionary of functions to apply to filter invalid grasps.\n\n Notes\n -----\n Required configuration dictionary parameters are specified in\n Other Parameters.\n\n Other Parameters\n ----------------\n num_grasp_samples : int\n Number of grasps to sample.\n gripper_width : float, optional\n Width of the gripper in meters.\n logging_dir : str, optional\n Directory in which to save the sampled grasps and input images.\n \"\"\"\n GraspingPolicy.__init__(self, config)\n self._parse_config()\n self._filters = filters\n\n def _parse_config(self):\n \"\"\"Parses the parameters of the policy.\"\"\"\n self._num_grasp_samples = self.config[\"sampling\"][\"num_grasp_samples\"]\n self._max_grasps_filter = 1\n if \"max_grasps_filter\" in self.config:\n self._max_grasps_filter = self.config[\"max_grasps_filter\"]\n self._gripper_width = np.inf\n if \"gripper_width\" in self.config:\n self._gripper_width = self.config[\"gripper_width\"]\n\n def select(self, grasps, q_value):\n \"\"\"Selects the grasp with the highest probability of success.\n\n Can override for alternate policies (e.g. epsilon greedy).\n\n Parameters\n ----------\n grasps : list\n Python list of :obj:`gqcnn.grasping.Grasp2D` or\n :obj:`gqcnn.grasping.SuctionPoint2D` grasps to select from.\n q_values : list\n Python list of associated q-values.\n\n Returns\n -------\n :obj:`gqcnn.grasping.Grasp2D` or :obj:`gqcnn.grasping.SuctionPoint2D`\n Grasp with highest probability of success .\n \"\"\"\n # Sort grasps.\n num_grasps = len(grasps)\n grasps_and_predictions = zip(np.arange(num_grasps), q_value)\n grasps_and_predictions = sorted(grasps_and_predictions,\n key=lambda x: x[1],\n reverse=True)\n\n # Return top grasps.\n if self._filters is None:\n return grasps_and_predictions[0][0]\n\n # Filter grasps.\n self._logger.info(\"Filtering grasps\")\n i = 0\n while i < self._max_grasps_filter and i < len(grasps_and_predictions):\n index = grasps_and_predictions[i][0]\n grasp = grasps[index]\n valid = True\n for filter_name, is_valid in self._filters.items():\n valid = is_valid(grasp)\n self._logger.debug(\"Grasp {} filter {} valid: {}\".format(\n i, filter_name, valid))\n if not valid:\n valid = False\n break\n if valid:\n return index\n i += 1\n raise NoValidGraspsException(\"No grasps satisfied filters\")\n\n def _action(self, state):\n \"\"\"Plans the grasp with the highest probability of success on\n the given RGB-D image.\n\n Attributes\n ----------\n state : :obj:`RgbdImageState`\n Image to plan grasps on.\n\n Returns\n -------\n :obj:`GraspAction`\n Grasp to execute.\n \"\"\"\n # Check valid input.\n if not isinstance(state, RgbdImageState):\n raise ValueError(\"Must provide an RGB-D image state.\")\n\n # Parse state.\n rgbd_im = state.rgbd_im\n camera_intr = state.camera_intr\n segmask = state.segmask\n\n # Sample grasps.\n grasps = self._grasp_sampler.sample(\n rgbd_im,\n camera_intr,\n self._num_grasp_samples,\n segmask=segmask,\n visualize=self.config[\"vis\"][\"grasp_sampling\"],\n constraint_fn=self._grasp_constraint_fn,\n seed=None)\n num_grasps = len(grasps)\n if num_grasps == 0:\n self._logger.warning(\"No valid grasps could be found\")\n raise NoValidGraspsException()\n\n # Compute grasp quality.\n compute_start = time()\n q_values = self._grasp_quality_fn(state, grasps, params=self._config)\n self._logger.debug(\"Grasp evaluation took %.3f sec\" %\n (time() - compute_start))\n\n if self.config[\"vis\"][\"grasp_candidates\"]:\n # Display each grasp on the original image, colored by predicted\n # success.\n norm_q_values = (q_values - np.min(q_values)) / (np.max(q_values) -\n np.min(q_values))\n vis.figure(size=(GeneralConstants.FIGSIZE,\n GeneralConstants.FIGSIZE))\n vis.imshow(rgbd_im.depth,\n vmin=self.config[\"vis\"][\"vmin\"],\n vmax=self.config[\"vis\"][\"vmax\"])\n for grasp, q in zip(grasps, norm_q_values):\n vis.grasp(grasp,\n scale=1.0,\n grasp_center_size=10,\n show_center=False,\n show_axis=True,\n color=plt.cm.RdYlBu(q))\n vis.title(\"Sampled grasps\")\n filename = None\n if self._logging_dir is not None:\n filename = os.path.join(self._logging_dir,\n \"grasp_candidates.png\")\n vis.show(filename)\n\n # Select grasp.\n index = self.select(grasps, q_values)\n grasp = grasps[index]\n q_value = q_values[index]\n if self.config[\"vis\"][\"grasp_plan\"]:\n vis.figure()\n vis.imshow(rgbd_im.depth,\n vmin=self.config[\"vis\"][\"vmin\"],\n vmax=self.config[\"vis\"][\"vmax\"])\n vis.grasp(grasp, scale=2.0, show_axis=True)\n vis.title(\"Best Grasp: d=%.3f, q=%.3f\" % (grasp.depth, q_value))\n vis.show()\n\n return GraspAction(grasp, q_value, state.rgbd_im.depth)\n\n\nclass CrossEntropyRobustGraspingPolicy(GraspingPolicy):\n \"\"\"Optimizes a set of grasp candidates in image space using the\n cross entropy method.\n\n Cross entropy method (CEM):\n (1) sample an initial set of candidates\n (2) sort the candidates\n (3) fit a GMM to the top P%\n (4) re-sample grasps from the distribution\n (5) repeat steps 2-4 for K iters\n (6) return the best candidate from the final sample set\n \"\"\"\n\n def __init__(self, config, filters=None):\n \"\"\"\n Parameters\n ----------\n config : dict\n Python dictionary of policy configuration parameters.\n filters : dict\n Python dictionary of functions to apply to filter invalid grasps.\n\n Notes\n -----\n Required configuration dictionary parameters are specified in Other\n Parameters.\n\n Other Parameters\n ----------------\n num_seed_samples : int\n Number of candidate to sample in the initial set.\n num_gmm_samples : int\n Number of candidates to sample on each resampling from the GMMs.\n num_iters : int\n Number of sample-and-refit iterations of CEM.\n gmm_refit_p : float\n Top p-% of grasps used for refitting.\n gmm_component_frac : float\n Percentage of the elite set size used to determine number of GMM\n components.\n gmm_reg_covar : float\n Regularization parameters for GMM covariance matrix, enforces\n diversity of fitted distributions.\n deterministic : bool, optional\n Whether to set the random seed to enforce deterministic behavior.\n gripper_width : float, optional\n Width of the gripper in meters.\n \"\"\"\n GraspingPolicy.__init__(self, config)\n self._parse_config()\n self._filters = filters\n\n self._case_counter = 0\n\n def _parse_config(self):\n \"\"\"Parses the parameters of the policy.\"\"\"\n # Cross entropy method parameters.\n self._num_seed_samples = self.config[\"num_seed_samples\"]\n self._num_gmm_samples = self.config[\"num_gmm_samples\"]\n self._num_iters = self.config[\"num_iters\"]\n self._gmm_refit_p = self.config[\"gmm_refit_p\"]\n self._gmm_component_frac = self.config[\"gmm_component_frac\"]\n self._gmm_reg_covar = self.config[\"gmm_reg_covar\"]\n\n self._depth_gaussian_sigma = 0.0\n if \"depth_gaussian_sigma\" in self.config:\n self._depth_gaussian_sigma = self.config[\"depth_gaussian_sigma\"]\n\n self._max_grasps_filter = 1\n if \"max_grasps_filter\" in self.config:\n self._max_grasps_filter = self.config[\"max_grasps_filter\"]\n\n self._max_resamples_per_iteration = 100\n if \"max_resamples_per_iteration\" in self.config:\n self._max_resamples_per_iteration = self.config[\n \"max_resamples_per_iteration\"]\n\n self._max_approach_angle = np.inf\n if \"max_approach_angle\" in self.config:\n self._max_approach_angle = np.deg2rad(\n self.config[\"max_approach_angle\"])\n\n # Gripper parameters.\n self._seed = None\n if self.config[\"deterministic\"]:\n self._seed = GeneralConstants.SEED\n self._gripper_width = np.inf\n if \"gripper_width\" in self.config:\n self._gripper_width = self.config[\"gripper_width\"]\n\n # Affordance map visualization.\n self._vis_grasp_affordance_map = False\n if \"grasp_affordance_map\" in self.config[\"vis\"]:\n self._vis_grasp_affordance_map = self.config[\"vis\"][\n \"grasp_affordance_map\"]\n\n self._state_counter = 0 # Used for logging state data.\n\n def select(self, grasps, q_values):\n \"\"\"Selects the grasp with the highest probability of success.\n\n Can override for alternate policies (e.g. epsilon greedy).\n\n Parameters\n ----------\n grasps : list\n Python list of :obj:`gqcnn.grasping.Grasp2D` or\n :obj:`gqcnn.grasping.SuctionPoint2D` grasps to select from.\n q_values : list\n Python list of associated q-values.\n\n Returns\n -------\n :obj:`gqcnn.grasping.Grasp2D` or :obj:`gqcnn.grasping.SuctionPoint2D`\n Grasp with highest probability of success.\n \"\"\"\n # Sort.\n self._logger.info(\"Sorting grasps\")\n num_grasps = len(grasps)\n if num_grasps == 0:\n raise NoValidGraspsException(\"Zero grasps\")\n grasps_and_predictions = zip(np.arange(num_grasps), q_values)\n grasps_and_predictions = sorted(grasps_and_predictions,\n key=lambda x: x[1],\n reverse=True)\n\n # Return top grasps.\n if self._filters is None:\n return grasps_and_predictions[0][0]\n\n # Filter grasps.\n self._logger.info(\"Filtering grasps\")\n i = 0\n while i < self._max_grasps_filter and i < len(grasps_and_predictions):\n index = grasps_and_predictions[i][0]\n grasp = grasps[index]\n valid = True\n for filter_name, is_valid in self._filters.items():\n valid = is_valid(grasp)\n self._logger.debug(\"Grasp {} filter {} valid: {}\".format(\n i, filter_name, valid))\n if not valid:\n valid = False\n break\n if valid:\n return index\n i += 1\n raise NoValidGraspsException(\"No grasps satisfied filters\")\n\n def _mask_predictions(self, pred_map, segmask):\n self._logger.info(\"Masking predictions...\")\n seg_pred_mismatch_msg = (\"Prediction map shape {} does not match shape\"\n \" of segmask {}.\")\n assert pred_map.shape == segmask.shape, seg_pred_mismatch_msg.format(\n pred_map.shape, segmask.shape)\n preds_masked = np.zeros_like(pred_map)\n nonzero_ind = np.where(segmask > 0)\n preds_masked[nonzero_ind] = pred_map[nonzero_ind]\n return preds_masked\n\n def _gen_grasp_affordance_map(self, state, stride=1):\n self._logger.info(\"Generating grasp affordance map...\")\n\n # Generate grasps at points to evaluate (this is just the interface to\n # `GraspQualityFunction`).\n crop_candidate_start_time = time()\n point_cloud_im = state.camera_intr.deproject_to_image(\n state.rgbd_im.depth)\n normal_cloud_im = point_cloud_im.normal_cloud_im()\n\n q_vals = []\n gqcnn_recep_h_half = self._grasp_quality_fn.gqcnn_recep_height // 2\n gqcnn_recep_w_half = self._grasp_quality_fn.gqcnn_recep_width // 2\n im_h = state.rgbd_im.height\n im_w = state.rgbd_im.width\n for i in range(gqcnn_recep_h_half - 1, im_h - gqcnn_recep_h_half,\n stride):\n grasps = []\n for j in range(gqcnn_recep_w_half - 1, im_w - gqcnn_recep_w_half,\n stride):\n # TODO(vsatish): Find a better way to find policy type.\n if self.config[\"sampling\"][\"type\"] == \"suction\":\n grasps.append(\n SuctionPoint2D(Point(np.array([j, i])),\n axis=-normal_cloud_im[i, j],\n depth=state.rgbd_im.depth[i, j],\n camera_intr=state.camera_intr))\n else:\n raise NotImplementedError(\n \"Parallel Jaw Grasp Affordance Maps Not Supported!\")\n q_vals.extend(self._grasp_quality_fn(state, grasps))\n self._logger.info(\n \"Generating crop grasp candidates took {} sec.\".format(\n time() - crop_candidate_start_time))\n\n # Mask out predictions not in the segmask (we don't really care about\n # them).\n pred_map = np.array(q_vals).reshape(\n (im_h - gqcnn_recep_h_half * 2) // stride + 1,\n (im_w - gqcnn_recep_w_half * 2) // stride + 1)\n # TODO(vsatish): Don't access the raw data like this!\n tf_segmask = state.segmask.crop(im_h - gqcnn_recep_h_half * 2,\n im_w - gqcnn_recep_w_half * 2).resize(\n 1.0 / stride,\n interp=\"nearest\")._data.squeeze()\n if tf_segmask.shape != pred_map.shape:\n new_tf_segmask = np.zeros_like(pred_map)\n smaller_i = min(pred_map.shape[0], tf_segmask.shape[0])\n smaller_j = min(pred_map.shape[1], tf_segmask.shape[1])\n new_tf_segmask[:smaller_i, :smaller_j] = tf_segmask[:smaller_i, :\n smaller_j]\n tf_segmask = new_tf_segmask\n pred_map_masked = self._mask_predictions(pred_map, tf_segmask)\n return pred_map_masked\n\n def _plot_grasp_affordance_map(self,\n state,\n affordance_map,\n stride=1,\n grasps=None,\n q_values=None,\n plot_max=True,\n title=None,\n scale=1.0,\n save_fname=None,\n save_path=None):\n gqcnn_recep_h_half = self._grasp_quality_fn.gqcnn_recep_height // 2\n gqcnn_recep_w_half = self._grasp_quality_fn.gqcnn_recep_width // 2\n im_h = state.rgbd_im.height\n im_w = state.rgbd_im.width\n\n # Plot.\n vis.figure()\n tf_depth_im = state.rgbd_im.depth.crop(\n im_h - gqcnn_recep_h_half * 2,\n im_w - gqcnn_recep_w_half * 2).resize(1.0 / stride,\n interp=\"nearest\")\n vis.imshow(tf_depth_im)\n plt.imshow(affordance_map, cmap=plt.cm.RdYlGn, alpha=0.3)\n if grasps is not None:\n grasps = copy.deepcopy(grasps)\n for grasp, q in zip(grasps, q_values):\n grasp.center.data[0] -= gqcnn_recep_w_half\n grasp.center.data[1] -= gqcnn_recep_h_half\n vis.grasp(grasp,\n scale=scale,\n show_center=False,\n show_axis=True,\n color=plt.cm.RdYlGn(q))\n if plot_max:\n affordance_argmax = np.unravel_index(np.argmax(affordance_map),\n affordance_map.shape)\n plt.scatter(affordance_argmax[1],\n affordance_argmax[0],\n c=\"black\",\n marker=\".\",\n s=scale * 25)\n if title is not None:\n vis.title(title)\n if save_path is not None:\n save_path = os.path.join(save_path, save_fname)\n vis.show(save_path)\n\n def action_set(self, state):\n \"\"\"Plan a set of grasps with the highest probability of success on\n the given RGB-D image.\n\n Parameters\n ----------\n state : :obj:`RgbdImageState`\n Image to plan grasps on.\n\n Returns\n -------\n python list of :obj:`gqcnn.grasping.Grasp2D` or :obj:`gqcnn.grasping.SuctionPoint2D` # noqa E501\n Grasps to execute.\n \"\"\"\n # Check valid input.\n if not isinstance(state, RgbdImageState):\n raise ValueError(\"Must provide an RGB-D image state.\")\n\n state_output_dir = None\n if self._logging_dir is not None:\n state_output_dir = os.path.join(\n self._logging_dir,\n \"state_{}\".format(str(self._state_counter).zfill(5)))\n if not os.path.exists(state_output_dir):\n os.makedirs(state_output_dir)\n self._state_counter += 1\n\n # Parse state.\n seed_set_start = time()\n rgbd_im = state.rgbd_im\n depth_im = rgbd_im.depth\n camera_intr = state.camera_intr\n segmask = state.segmask\n\n if self._depth_gaussian_sigma > 0:\n depth_im_filtered = depth_im.apply(\n snf.gaussian_filter, sigma=self._depth_gaussian_sigma)\n else:\n depth_im_filtered = depth_im\n point_cloud_im = camera_intr.deproject_to_image(depth_im_filtered)\n normal_cloud_im = point_cloud_im.normal_cloud_im()\n\n # Visualize grasp affordance map.\n if self._vis_grasp_affordance_map:\n grasp_affordance_map = self._gen_grasp_affordance_map(state)\n self._plot_grasp_affordance_map(state,\n grasp_affordance_map,\n title=\"Grasp Affordance Map\",\n save_fname=\"affordance_map.png\",\n save_path=state_output_dir)\n\n if \"input_images\" in self.config[\"vis\"] and self.config[\"vis\"][\n \"input_images\"]:\n vis.figure()\n vis.subplot(1, 2, 1)\n vis.imshow(depth_im)\n vis.title(\"Depth\")\n vis.subplot(1, 2, 2)\n vis.imshow(segmask)\n vis.title(\"Segmask\")\n filename = None\n if self._logging_dir is not None:\n filename = os.path.join(self._logging_dir, \"input_images.png\")\n vis.show(filename)\n\n # Sample grasps.\n self._logger.info(\"Sampling seed set\")\n grasps = self._grasp_sampler.sample(\n rgbd_im,\n camera_intr,\n self._num_seed_samples,\n segmask=segmask,\n visualize=self.config[\"vis\"][\"grasp_sampling\"],\n constraint_fn=self._grasp_constraint_fn,\n seed=self._seed)\n num_grasps = len(grasps)\n if num_grasps == 0:\n self._logger.warning(\"No valid grasps could be found\")\n raise NoValidGraspsException()\n\n grasp_type = \"parallel_jaw\"\n if isinstance(grasps[0], SuctionPoint2D):\n grasp_type = \"suction\"\n elif isinstance(grasps[0], MultiSuctionPoint2D):\n grasp_type = \"multi_suction\"\n\n self._logger.info(\"Sampled %d grasps\" % (len(grasps)))\n self._logger.info(\"Computing the seed set took %.3f sec\" %\n (time() - seed_set_start))\n\n # Iteratively refit and sample.\n for j in range(self._num_iters):\n self._logger.info(\"CEM iter %d\" % (j))\n\n # Predict grasps.\n predict_start = time()\n q_values = self._grasp_quality_fn(state,\n grasps,\n params=self._config)\n self._logger.info(\"Prediction took %.3f sec\" %\n (time() - predict_start))\n\n # Sort grasps.\n resample_start = time()\n q_values_and_indices = zip(q_values, np.arange(num_grasps))\n q_values_and_indices = sorted(q_values_and_indices,\n key=lambda x: x[0],\n reverse=True)\n\n if self.config[\"vis\"][\"grasp_candidates\"]:\n # Display each grasp on the original image, colored by\n # predicted success.\n # norm_q_values = ((q_values - np.min(q_values)) /\n # (np.max(q_values) - np.min(q_values)))\n norm_q_values = q_values\n title = \"Sampled Grasps Iter %d\" % (j)\n if self._vis_grasp_affordance_map:\n self._plot_grasp_affordance_map(\n state,\n grasp_affordance_map,\n grasps=grasps,\n q_values=norm_q_values,\n scale=2.0,\n title=title,\n save_fname=\"cem_iter_{}.png\".format(j),\n save_path=state_output_dir)\n display_grasps_and_q_values = zip(grasps, q_values)\n display_grasps_and_q_values = sorted(\n display_grasps_and_q_values, key=lambda x: x[1])\n vis.figure(size=(GeneralConstants.FIGSIZE,\n GeneralConstants.FIGSIZE))\n vis.imshow(rgbd_im.depth,\n vmin=self.config[\"vis\"][\"vmin\"],\n vmax=self.config[\"vis\"][\"vmax\"])\n for grasp, q in display_grasps_and_q_values:\n vis.grasp(grasp,\n scale=2.0,\n jaw_width=2.0,\n show_center=False,\n show_axis=True,\n color=plt.cm.RdYlBu(q))\n vis.title(\"Sampled grasps iter %d\" % (j))\n filename = None\n if self._logging_dir is not None:\n filename = os.path.join(self._logging_dir,\n \"cem_iter_%d.png\" % (j))\n vis.show(filename)\n\n # Fit elite set.\n elite_start = time()\n num_refit = max(int(np.ceil(self._gmm_refit_p * num_grasps)), 1)\n elite_q_values = [i[0] for i in q_values_and_indices[:num_refit]]\n elite_grasp_indices = [\n i[1] for i in q_values_and_indices[:num_refit]\n ]\n elite_grasps = [grasps[i] for i in elite_grasp_indices]\n elite_grasp_arr = np.array([g.feature_vec for g in elite_grasps])\n\n if self.config[\"vis\"][\"elite_grasps\"]:\n # Display each grasp on the original image, colored by\n # predicted success.\n norm_q_values = (elite_q_values - np.min(elite_q_values)) / (\n np.max(elite_q_values) - np.min(elite_q_values))\n vis.figure(size=(GeneralConstants.FIGSIZE,\n GeneralConstants.FIGSIZE))\n vis.imshow(rgbd_im.depth,\n vmin=self.config[\"vis\"][\"vmin\"],\n vmax=self.config[\"vis\"][\"vmax\"])\n for grasp, q in zip(elite_grasps, norm_q_values):\n vis.grasp(grasp,\n scale=1.5,\n show_center=False,\n show_axis=True,\n color=plt.cm.RdYlBu(q))\n vis.title(\"Elite grasps iter %d\" % (j))\n filename = None\n if self._logging_dir is not None:\n filename = os.path.join(self._logging_dir,\n \"elite_set_iter_%d.png\" % (j))\n vis.show(filename)\n\n # Normalize elite set.\n elite_grasp_mean = np.mean(elite_grasp_arr, axis=0)\n elite_grasp_std = np.std(elite_grasp_arr, axis=0)\n elite_grasp_std[elite_grasp_std == 0] = 1e-6\n elite_grasp_arr = (elite_grasp_arr -\n elite_grasp_mean) / elite_grasp_std\n self._logger.info(\"Elite set computation took %.3f sec\" %\n (time() - elite_start))\n\n # Fit a GMM to the top samples.\n num_components = max(\n int(np.ceil(self._gmm_component_frac * num_refit)), 1)\n uniform_weights = (1.0 / num_components) * np.ones(num_components)\n gmm = GaussianMixture(n_components=num_components,\n weights_init=uniform_weights,\n reg_covar=self._gmm_reg_covar)\n train_start = time()\n gmm.fit(elite_grasp_arr)\n self._logger.info(\"GMM fitting with %d components took %.3f sec\" %\n (num_components, time() - train_start))\n\n # Sample the next grasps.\n grasps = []\n loop_start = time()\n num_tries = 0\n while (len(grasps) < self._num_gmm_samples\n and num_tries < self._max_resamples_per_iteration):\n # Sample from GMM.\n sample_start = time()\n grasp_vecs, _ = gmm.sample(n_samples=self._num_gmm_samples)\n grasp_vecs = elite_grasp_std * grasp_vecs + elite_grasp_mean\n self._logger.info(\"GMM sampling took %.3f sec\" %\n (time() - sample_start))\n\n # Convert features to grasps and store if in segmask.\n for k, grasp_vec in enumerate(grasp_vecs):\n feature_start = time()\n if grasp_type == \"parallel_jaw\":\n # Form grasp object.\n grasp = Grasp2D.from_feature_vec(\n grasp_vec,\n width=self._gripper_width,\n camera_intr=camera_intr)\n elif grasp_type == \"suction\":\n # Read depth and approach axis.\n u = int(min(max(grasp_vec[1], 0), depth_im.height - 1))\n v = int(min(max(grasp_vec[0], 0), depth_im.width - 1))\n grasp_depth = depth_im[u, v, 0]\n\n # Approach axis.\n grasp_axis = -normal_cloud_im[u, v]\n\n # Form grasp object.\n grasp = SuctionPoint2D.from_feature_vec(\n grasp_vec,\n camera_intr=camera_intr,\n depth=grasp_depth,\n axis=grasp_axis)\n elif grasp_type == \"multi_suction\":\n # Read depth and approach axis.\n u = int(min(max(grasp_vec[1], 0), depth_im.height - 1))\n v = int(min(max(grasp_vec[0], 0), depth_im.width - 1))\n grasp_depth = depth_im[u, v]\n\n # Approach_axis.\n grasp_axis = -normal_cloud_im[u, v]\n\n # Form grasp object.\n grasp = MultiSuctionPoint2D.from_feature_vec(\n grasp_vec,\n camera_intr=camera_intr,\n depth=grasp_depth,\n axis=grasp_axis)\n self._logger.debug(\"Feature vec took %.5f sec\" %\n (time() - feature_start))\n\n bounds_start = time()\n # Check in bounds.\n if (state.segmask is None or\n (grasp.center.y >= 0\n and grasp.center.y < state.segmask.height\n and grasp.center.x >= 0\n and grasp.center.x < state.segmask.width\n and np.any(state.segmask[int(grasp.center.y),\n int(grasp.center.x)] != 0)\n and grasp.approach_angle < self._max_approach_angle)\n and (self._grasp_constraint_fn is None\n or self._grasp_constraint_fn(grasp))):\n\n # Check validity according to filters.\n grasps.append(grasp)\n self._logger.debug(\"Bounds took %.5f sec\" %\n (time() - bounds_start))\n num_tries += 1\n\n # Check num grasps.\n num_grasps = len(grasps)\n if num_grasps == 0:\n self._logger.warning(\"No valid grasps could be found\")\n raise NoValidGraspsException()\n self._logger.info(\"Resample loop took %.3f sec\" %\n (time() - loop_start))\n self._logger.info(\"Resampling took %.3f sec\" %\n (time() - resample_start))\n\n # Predict final set of grasps.\n predict_start = time()\n q_values = self._grasp_quality_fn(state, grasps, params=self._config)\n self._logger.info(\"Final prediction took %.3f sec\" %\n (time() - predict_start))\n\n if self.config[\"vis\"][\"grasp_candidates\"]:\n # Display each grasp on the original image, colored by predicted\n # success.\n # norm_q_values = ((q_values - np.min(q_values)) /\n # (np.max(q_values) - np.min(q_values)))\n norm_q_values = q_values\n title = \"Final Sampled Grasps\"\n if self._vis_grasp_affordance_map:\n self._plot_grasp_affordance_map(\n state,\n grasp_affordance_map,\n grasps=grasps,\n q_values=norm_q_values,\n scale=2.0,\n title=title,\n save_fname=\"final_sampled_grasps.png\",\n save_path=state_output_dir)\n display_grasps_and_q_values = zip(grasps, q_values)\n display_grasps_and_q_values = sorted(display_grasps_and_q_values,\n key=lambda x: x[1])\n vis.figure(size=(GeneralConstants.FIGSIZE,\n GeneralConstants.FIGSIZE))\n vis.imshow(rgbd_im.depth,\n vmin=self.config[\"vis\"][\"vmin\"],\n vmax=self.config[\"vis\"][\"vmax\"])\n for grasp, q in display_grasps_and_q_values:\n vis.grasp(grasp,\n scale=2.0,\n jaw_width=2.0,\n show_center=False,\n show_axis=True,\n color=plt.cm.RdYlBu(q))\n vis.title(\"Sampled grasps iter %d\" % (j))\n filename = None\n if self._logging_dir is not None:\n filename = os.path.join(self._logging_dir,\n \"cem_iter_%d.png\" % (j))\n vis.show(filename)\n\n return grasps, q_values\n\n def _action(self, state):\n \"\"\"Plans the grasp with the highest probability of success on\n the given RGB-D image.\n\n Attributes\n ----------\n state : :obj:`RgbdImageState`\n Image to plan grasps on.\n\n Returns\n -------\n :obj:`GraspAction`\n Grasp to execute.\n \"\"\"\n # Parse state.\n rgbd_im = state.rgbd_im\n depth_im = rgbd_im.depth\n # camera_intr = state.camera_intr\n # segmask = state.segmask\n\n # Plan grasps.\n grasps, q_values = self.action_set(state)\n\n # Select grasp.\n index = self.select(grasps, q_values)\n grasp = grasps[index]\n q_value = q_values[index]\n if self.config[\"vis\"][\"grasp_plan\"]:\n title = \"Best Grasp: d=%.3f, q=%.3f\" % (grasp.depth, q_value)\n vis.figure()\n vis.imshow(depth_im,\n vmin=self.config[\"vis\"][\"vmin\"],\n vmax=self.config[\"vis\"][\"vmax\"])\n vis.grasp(grasp,\n scale=5.0,\n show_center=False,\n show_axis=True,\n jaw_width=1.0,\n grasp_axis_width=0.2)\n vis.title(title)\n filename = None\n if self._logging_dir is not None:\n filename = os.path.join(self._logging_dir, \"planned_grasp.png\")\n vis.show(filename)\n\n # Form return image.\n image = depth_im\n if isinstance(self._grasp_quality_fn, GQCnnQualityFunction):\n image_arr, _ = self._grasp_quality_fn.grasps_to_tensors([grasp],\n state)\n image = DepthImage(image_arr[0, ...], frame=rgbd_im.frame)\n\n # Return action.\n action = GraspAction(grasp, q_value, image)\n return action\n\n\nclass QFunctionRobustGraspingPolicy(CrossEntropyRobustGraspingPolicy):\n \"\"\"Optimizes a set of antipodal grasp candidates in image space using the\n cross entropy method with a GQ-CNN that estimates the Q-function\n for use in Q-learning.\n\n Notes\n -----\n Required configuration parameters are specified in Other Parameters.\n\n Other Parameters\n ----------------\n reinit_pc1 : bool\n Whether or not to reinitialize the pc1 layer of the GQ-CNN.\n reinit_fc3: bool\n Whether or not to reinitialize the fc3 layer of the GQ-CNN.\n reinit_fc4: bool\n Whether or not to reinitialize the fc4 layer of the GQ-CNN.\n reinit_fc5: bool\n Whether or not to reinitialize the fc5 layer of the GQ-CNN.\n num_seed_samples : int\n Number of candidate to sample in the initial set.\n num_gmm_samples : int\n Number of candidates to sample on each resampling from the GMMs.\n num_iters : int\n Number of sample-and-refit iterations of CEM.\n gmm_refit_p : float\n Top p-% of grasps used for refitting.\n gmm_component_frac : float\n Percentage of the elite set size used to determine number of GMM\n components.\n gmm_reg_covar : float\n Regularization parameters for GMM covariance matrix, enforces diversity\n of fitted distributions.\n deterministic : bool, optional\n Whether to set the random seed to enforce deterministic behavior.\n gripper_width : float, optional\n Width of the gripper in meters.\n \"\"\"\n\n def __init__(self, config):\n CrossEntropyRobustGraspingPolicy.__init__(self, config)\n QFunctionRobustGraspingPolicy._parse_config(self)\n self._setup_gqcnn()\n\n def _parse_config(self):\n \"\"\"Parses the parameters of the policy.\"\"\"\n self._reinit_pc1 = self.config[\"reinit_pc1\"]\n self._reinit_fc3 = self.config[\"reinit_fc3\"]\n self._reinit_fc4 = self.config[\"reinit_fc4\"]\n self._reinit_fc5 = self.config[\"reinit_fc5\"]\n\n def _setup_gqcnn(self):\n \"\"\"Sets up the GQ-CNN.\"\"\"\n # Close existing session (from superclass initializer).\n self.gqcnn.close_session()\n\n # Check valid output size.\n if self.gqcnn.fc5_out_size != 1 and not self._reinit_fc5:\n raise ValueError(\"Q function must return scalar values\")\n\n # Reinitialize layers\n if self._reinit_fc5:\n self.gqcnn.fc5_out_size = 1\n\n # TODO(jmahler): Implement reinitialization of pc0.\n self.gqcnn.reinitialize_layers(self._reinit_fc3, self._reinit_fc4,\n self._reinit_fc5)\n self.gqcnn.initialize_network(add_softmax=False)\n\n\nclass EpsilonGreedyQFunctionRobustGraspingPolicy(QFunctionRobustGraspingPolicy\n ):\n \"\"\"Optimizes a set of antipodal grasp candidates in image space\n using the cross entropy method with a GQ-CNN that estimates the\n Q-function for use in Q-learning, and chooses a random antipodal\n grasp with probability epsilon.\n\n Notes\n -----\n Required configuration parameters are specified in Other Parameters.\n\n Other Parameters\n ----------------\n epsilon : float\n \"\"\"\n\n def __init__(self, config):\n QFunctionRobustGraspingPolicy.__init__(self, config)\n self._parse_config()\n\n def _parse_config(self):\n \"\"\"Parses the parameters of the policy.\"\"\"\n self._epsilon = self.config[\"epsilon\"]\n\n @property\n def epsilon(self):\n return self._epsilon\n\n @epsilon.setter\n def epsilon(self, val):\n self._epsilon = val\n\n def greedy_action(self, state):\n \"\"\"Plans the grasp with the highest probability of success on\n the given RGB-D image.\n\n Attributes\n ----------\n state : :obj:`RgbdImageState`\n Image to plan grasps on.\n\n Returns\n -------\n :obj:`GraspAction`\n Grasp to execute.\n \"\"\"\n return CrossEntropyRobustGraspingPolicy.action(self, state)\n\n def _action(self, state):\n \"\"\"Plans the grasp with the highest probability of success on\n the given RGB-D image.\n\n Attributes\n ----------\n state : :obj:`RgbdImageState`\n Image to plan grasps on.\n\n Returns\n -------\n :obj:`GraspAction`\n Grasp to execute.\n \"\"\"\n # Take the greedy action with prob 1-epsilon.\n if np.random.rand() > self.epsilon:\n self._logger.debug(\"Taking greedy action\")\n return CrossEntropyRobustGraspingPolicy.action(self, state)\n\n # Otherwise take a random action.\n self._logger.debug(\"Taking random action\")\n\n # Check valid input.\n if not isinstance(state, RgbdImageState):\n raise ValueError(\"Must provide an RGB-D image state.\")\n\n # Parse state.\n rgbd_im = state.rgbd_im\n camera_intr = state.camera_intr\n segmask = state.segmask\n\n # Sample random antipodal grasps.\n grasps = self._grasp_sampler.sample(\n rgbd_im,\n camera_intr,\n self._num_seed_samples,\n segmask=segmask,\n visualize=self.config[\"vis\"][\"grasp_sampling\"],\n constraint_fn=self._grasp_constraint_fn,\n seed=self._seed)\n\n num_grasps = len(grasps)\n if num_grasps == 0:\n self._logger.warning(\"No valid grasps could be found\")\n raise NoValidGraspsException()\n\n # Choose a grasp uniformly at random.\n grasp_ind = np.random.choice(num_grasps, size=1)[0]\n grasp = grasps[grasp_ind]\n depth = grasp.depth\n\n # Create transformed image.\n image_tensor, pose_tensor = self.grasps_to_tensors([grasp], state)\n image = DepthImage(image_tensor[0, ...])\n\n # Predict prob success.\n output_arr = self.gqcnn.predict(image_tensor, pose_tensor)\n q_value = output_arr[0, -1]\n\n # Visualize planned grasp.\n if self.config[\"vis\"][\"grasp_plan\"]:\n scale_factor = self.gqcnn.im_width / self._crop_width\n scaled_camera_intr = camera_intr.resize(scale_factor)\n vis_grasp = Grasp2D(Point(image.center),\n 0.0,\n depth,\n width=self._gripper_width,\n camera_intr=scaled_camera_intr)\n vis.figure()\n vis.imshow(image)\n vis.grasp(vis_grasp, scale=1.5, show_center=False, show_axis=True)\n vis.title(\"Best Grasp: d=%.3f, q=%.3f\" % (depth, q_value))\n vis.show()\n\n # Return action.\n return GraspAction(grasp, q_value, image)\n\n\nclass CompositeGraspingPolicy(Policy):\n \"\"\"Grasping policy composed of multiple sub-policies.\n\n Attributes\n ----------\n policies : dict mapping str to `gqcnn.GraspingPolicy`\n Key-value dict mapping policy names to grasping policies.\n \"\"\"\n\n def __init__(self, policies):\n self._policies = policies\n self._logger = Logger.get_logger(self.__class__.__name__,\n log_file=None,\n global_log_file=True)\n\n @property\n def policies(self):\n return self._policies\n\n def subpolicy(self, name):\n return self._policies[name]\n\n def set_constraint_fn(self, constraint_fn):\n for policy in self._policies:\n policy.set_constraint_fn(constraint_fn)\n\n\nclass PriorityCompositeGraspingPolicy(CompositeGraspingPolicy):\n\n def __init__(self, policies, priority_list):\n # Check validity.\n for name in priority_list:\n if str(name) not in policies:\n raise ValueError(\n \"Policy named %s is not in the list of policies!\" % (name))\n\n self._priority_list = priority_list\n CompositeGraspingPolicy.__init__(self, policies)\n\n @property\n def priority_list(self):\n return self._priority_list\n\n def action(self, state, policy_subset=None, min_q_value=-1.0):\n \"\"\"Returns an action for a given state.\n \"\"\"\n action = None\n i = 0\n max_q = min_q_value\n\n while action is None or (max_q <= min_q_value\n and i < len(self._priority_list)):\n name = self._priority_list[i]\n if policy_subset is not None and name not in policy_subset:\n i += 1\n continue\n self._logger.info(\"Planning action for sub-policy {}\".format(name))\n try:\n action = self.policies[name].action(state)\n action.policy_name = name\n max_q = action.q_value\n except NoValidGraspsException:\n pass\n i += 1\n if action is None:\n raise NoValidGraspsException()\n return action\n\n def action_set(self, state, policy_subset=None, min_q_value=-1.0):\n \"\"\"Returns an action for a given state.\n \"\"\"\n actions = None\n q_values = None\n i = 0\n max_q = min_q_value\n while actions is None or (max_q <= min_q_value\n and i < len(self._priority_list)):\n name = self._priority_list[i]\n if policy_subset is not None and name not in policy_subset:\n i += 1\n continue\n self._logger.info(\n \"Planning action set for sub-policy {}\".format(name))\n try:\n actions, q_values = self.policies[name].action_set(state)\n for action in actions:\n action.policy_name = name\n max_q = np.max(q_values)\n except NoValidGraspsException:\n pass\n i += 1\n if actions is None:\n raise NoValidGraspsException()\n return actions, q_values\n\n\nclass GreedyCompositeGraspingPolicy(CompositeGraspingPolicy):\n\n def __init__(self, policies):\n CompositeGraspingPolicy.__init__(self, policies)\n\n def action(self, state, policy_subset=None, min_q_value=-1.0):\n \"\"\"Returns an action for a given state.\"\"\"\n # Compute all possible actions.\n actions = []\n for name, policy in self.policies.items():\n if policy_subset is not None and name not in policy_subset:\n continue\n try:\n action = policy.action(state)\n action.policy_name = name\n actions.append()\n except NoValidGraspsException:\n pass\n\n if len(actions) == 0:\n raise NoValidGraspsException()\n\n # Rank based on q value.\n actions = sorted(actions, key=lambda x: x.q_value, reverse=True)\n return actions[0]\n\n def action_set(self, state, policy_subset=None, min_q_value=-1.0):\n \"\"\"Returns an action for a given state.\"\"\"\n actions = []\n q_values = []\n for name, policy in self.policies.items():\n if policy_subset is not None and name not in policy_subset:\n continue\n try:\n action_set, q_vals = self.policies[name].action_set(state)\n for action in action_set:\n action.policy_name = name\n actions.extend(action_set)\n q_values.extend(q_vals)\n except NoValidGraspsException:\n continue\n if actions is None:\n raise NoValidGraspsException()\n return actions, q_values\n" }, { "path": "gqcnn/model/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nFactory functions to obtain `GQCNN`/`FCGQCNN` class based on backend.\nAuthor: Vishal Satish\n\"\"\"\nfrom .tf import GQCNNTF, FCGQCNNTF\n\nfrom autolab_core import Logger\n\n\ndef get_gqcnn_model(backend=\"tf\", verbose=True):\n \"\"\"Get the GQ-CNN model for the provided backend.\n\n Note:\n Currently only TensorFlow is supported.\n\n Parameters\n ----------\n backend : str\n The backend to use, currently only \"tf\" is supported.\n verbose : bool\n Whether or not to log initialization output to `stdout`.\n\n Returns\n -------\n :obj:`gqcnn.model.tf.GQCNNTF`\n GQ-CNN model with TensorFlow backend.\n \"\"\"\n\n # Set up logger.\n logger = Logger.get_logger(\"GQCNNModelFactory\", silence=(not verbose))\n\n # Return desired GQ-CNN instance based on backend.\n if backend == \"tf\":\n logger.info(\"Initializing GQ-CNN with Tensorflow as backend...\")\n return GQCNNTF\n else:\n raise ValueError(\"Invalid backend: {}\".format(backend))\n\n\ndef get_fc_gqcnn_model(backend=\"tf\", verbose=True):\n \"\"\"Get the FC-GQ-CNN model for the provided backend.\n\n Note:\n Currently only TensorFlow is supported.\n\n Parameters\n ----------\n backend : str\n The backend to use, currently only \"tf\" is supported.\n verbose : bool\n Whether or not to log initialization output to `stdout`.\n\n Returns\n -------\n :obj:`gqcnn.model.tf.FCGQCNNTF`\n FC-GQ-CNN model with TensorFlow backend.\n \"\"\"\n\n # Set up logger.\n logger = Logger.get_logger(\"FCGQCNNModelFactory\", silence=(not verbose))\n\n # Return desired Fully-Convolutional GQ-CNN instance based on backend.\n if backend == \"tf\":\n logger.info(\"Initializing FC-GQ-CNN with Tensorflow as backend...\")\n return FCGQCNNTF\n else:\n raise ValueError(\"Invalid backend: {}\".format(backend))\n" }, { "path": "gqcnn/model/tf/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\"\"\"\nfrom .network_tf import GQCNNTF\nfrom .fc_network_tf import FCGQCNNTF\n\n__all__ = [\"GQCNNTF\", \"FCGQCNNTF\"]\n" }, { "path": "gqcnn/model/tf/fc_network_tf.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nFC-GQ-CNN network implemented in Tensorflow.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nfrom collections import OrderedDict\nimport json\nimport os\n\nimport tensorflow as tf\n\nfrom .network_tf import GQCNNTF\nfrom ...utils import TrainingMode, InputDepthMode, GQCNNFilenames\n\n\nclass FCGQCNNTF(GQCNNTF):\n \"\"\"FC-GQ-CNN network implemented in Tensorflow.\n\n Note\n ----\n FC-GQ-CNNs are never directly trained, but instead a pre-trained GQ-CNN\n is converted to an FC-GQ-CNN at inference time.\n \"\"\"\n\n def __init__(self, gqcnn_config, fc_config, verbose=True, log_file=None):\n \"\"\"\n Parameters\n ----------\n gqcnn_config : dict\n Python dictionary of pre-trained GQ-CNN model configuration\n parameters.\n fc_config : dict\n Python dictionary of FC-GQ-CNN model configuration parameters.\n verbose : bool\n Whether or not to log model output to `stdout`.\n log_file : str\n If provided, model output will also be logged to this file.\n \"\"\"\n super(FCGQCNNTF, self).__init__(gqcnn_config, log_file=log_file)\n super(FCGQCNNTF, self)._parse_config(gqcnn_config)\n # We call the child `_parse_config` again (even though it gets called\n # in the parent `GQCNNTF` constructor) because the call to the parent\n # `_parse_config` overwrites the first call.\n self._parse_config(fc_config)\n\n # Check that conv layers of GQ-CNN were trained with VALID padding.\n for layer_name, layer_config in self._architecture[\"im_stream\"].items(\n ):\n if layer_config[\"type\"] == \"conv\":\n invalid_pad_msg = (\"GQ-CNN used for FC-GQ-CNN must have VALID\"\n \" padding for conv layers. Found layer: {}\"\n \" with padding: {}\")\n assert layer_config[\"pad\"] == \"VALID\", invalid_pad_msg.format(\n layer_name, layer_config[\"pad\"])\n\n @staticmethod\n def load(model_dir, fc_config, log_file=None):\n \"\"\"Load an FC-GQ-CNN from a pre-trained GQ-CNN.\n\n Parameters\n ----------\n model_dir : str\n Path to pre-trained GQ-CNN model.\n fc_config : dict\n Python dictionary of FC-GQ-CNN model configuration parameters.\n log_file : str\n If provided, model output will also be logged to this file.\n\n Returns\n -------\n :obj:`FCGQCNNTF`\n Initialized FC-GQ-CNN.\n \"\"\"\n config_file = os.path.join(model_dir, GQCNNFilenames.SAVED_CFG)\n with open(config_file) as data_file:\n train_config = json.load(data_file, object_pairs_hook=OrderedDict)\n\n gqcnn_config = train_config[\"gqcnn\"]\n\n # Initialize weights and Tensorflow network.\n fcgqcnn = FCGQCNNTF(gqcnn_config, fc_config, log_file=log_file)\n fcgqcnn.init_weights_file(\n os.path.join(model_dir, GQCNNFilenames.FINAL_MODEL))\n fcgqcnn.init_mean_and_std(model_dir)\n training_mode = train_config[\"training_mode\"]\n if training_mode == TrainingMode.CLASSIFICATION:\n fcgqcnn.initialize_network(add_softmax=True)\n elif training_mode == TrainingMode.REGRESSION:\n fcgqcnn.initialize_network()\n else:\n raise ValueError(\"Invalid training mode: {}\".format(training_mode))\n return fcgqcnn\n\n def _parse_config(self, cfg):\n # Override GQ-CNN image height and width.\n self._im_width = cfg[\"im_width\"]\n self._im_height = cfg[\"im_height\"]\n\n def _pack(self, dim_h, dim_w, data, vector=False):\n if vector:\n # First reshape vector into 3-dimensional tensor.\n reshaped = tf.reshape(data, tf.concat([[1, 1], tf.shape(data)], 0))\n\n # Then tile into tensor of shape dim_h x dim_w x `data.dim0`.\n packed = tf.tile(reshaped, [dim_h, dim_w, 1])\n else:\n # First reshape second dimension of tensor into 3-dimensional\n # tensor.\n reshaped = tf.reshape(\n data,\n tf.concat([tf.shape(data)[0:1], [1, 1],\n tf.shape(data)[1:]], 0))\n\n # Then tile into tensor of shape\n # bsize=1 x dim_h x dim_w x `data.dim1`.\n packed = tf.tile(reshaped, [1, dim_h, dim_w, 1])\n return packed\n\n def _build_fully_conv_layer(self,\n input_node,\n filter_dim,\n fc_name,\n final_fc_layer=False):\n self._logger.info(\n \"Converting fc layer {} to fully convolutional...\".format(fc_name))\n\n # Create new set of weights by reshaping fully connected layer weights.\n fcW = self._weights.weights[\"{}_weights\".format(fc_name)]\n convW = tf.Variable(tf.reshape(\n fcW,\n tf.concat([[filter_dim, filter_dim],\n [tf.shape(fcW)[0] // (filter_dim * filter_dim)],\n tf.shape(fcW)[1:]], 0)),\n name=\"{}_fully_conv_weights\".format(fc_name))\n self._weights.weights[\"{}_fully_conv_weights\".format(fc_name)] = convW\n\n # Get bias.\n convb = self._weights.weights[\"{}_bias\".format(fc_name)]\n\n # Compute conv out (note that we use padding=\"VALID\" here because we\n # want an output size of 1 x 1 x num_filts for the original input\n # size).\n convh = tf.nn.conv2d(input_node,\n convW,\n strides=[1, 1, 1, 1],\n padding=\"VALID\")\n\n # Pack bias into tensor of shape `tf.shape(convh)`.\n # TODO(vsatish): This is unnecessary and can be removed for potential\n # performance improvements.\n bias_packed = self._pack(tf.shape(convh)[1],\n tf.shape(convh)[2],\n convb,\n vector=True)\n\n # Add bias term.\n convh = convh + bias_packed\n\n # Apply activation.\n if not final_fc_layer:\n convh = self._leaky_relu(convh, alpha=self._relu_coeff)\n\n # Add output to feature_dict.\n self._feature_tensors[fc_name] = convh\n\n return convh\n\n def _build_fully_conv_merge_layer(self, input_node_im, input_node_pose,\n filter_dim, fc_name):\n self._logger.info(\n \"Converting fc merge layer {} to fully convolutional...\".format(\n fc_name))\n\n # Create new set of weights for image stream by reshaping\n # fully-connected layer weights.\n fcW_im = self._weights.weights[\"{}_input_1_weights\".format(fc_name)]\n convW = tf.Variable(tf.reshape(\n fcW_im,\n tf.concat([[filter_dim, filter_dim],\n [tf.shape(fcW_im)[0] // (filter_dim * filter_dim)],\n tf.shape(fcW_im)[1:]], 0)),\n name=\"{}_im_fully_conv_weights\".format(fc_name))\n self._weights.weights[\"{}_im_fully_conv_weights\".format(\n fc_name)] = convW\n\n # Compute im stream conv out (note that we use padding=\"VALID\" here\n # because we want an output size of 1 x 1 x num_filts for the original\n # input size).\n convh_im = tf.nn.conv2d(input_node_im,\n convW,\n strides=[1, 1, 1, 1],\n padding=\"VALID\")\n\n # Get pose stream fully-connected weights.\n fcW_pose = self._weights.weights[\"{}_input_2_weights\".format(fc_name)]\n\n # Compute matmul for pose stream.\n pose_out = tf.matmul(input_node_pose, fcW_pose)\n\n # Pack pose_out into a tensor of shape=tf.shape(convh_im).\n # TODO(vsatish): This is unnecessary and can be removed for potential\n # performance improvements.\n pose_packed = self._pack(\n tf.shape(convh_im)[1],\n tf.shape(convh_im)[2], pose_out)\n\n # Add the im and pose tensors.\n convh = convh_im + pose_packed\n\n # Pack bias.\n # TODO(vsatish): This is unnecessary and can be removed for potential\n # performance improvements.\n fc_bias = self._weights.weights[\"{}_bias\".format(fc_name)]\n bias_packed = self._pack(tf.shape(convh_im)[1],\n tf.shape(convh_im)[2],\n fc_bias,\n vector=True)\n\n # Add bias and apply activation.\n convh = self._leaky_relu(convh + bias_packed, alpha=self._relu_coeff)\n\n return convh\n\n def _build_im_stream(self,\n input_node,\n input_pose_node,\n input_height,\n input_width,\n input_channels,\n drop_rate,\n layers,\n only_stream=False):\n self._logger.info(\"Building Image Stream...\")\n\n if self._input_depth_mode == InputDepthMode.SUB:\n sub_mean = tf.constant(self._im_depth_sub_mean, dtype=tf.float32)\n sub_std = tf.constant(self._im_depth_sub_std, dtype=tf.float32)\n sub_im = tf.subtract(\n input_node,\n tf.tile(\n tf.reshape(input_pose_node, tf.constant((-1, 1, 1, 1))),\n tf.constant((1, input_height, input_width, 1))))\n norm_sub_im = tf.div(tf.subtract(sub_im, sub_mean), sub_std)\n input_node = norm_sub_im\n\n output_node = input_node\n prev_layer = \"start\" # Dummy placeholder.\n filter_dim = self._train_im_width\n last_index = len(layers) - 1\n for layer_index, (layer_name,\n layer_config) in enumerate(layers.items()):\n layer_type = layer_config[\"type\"]\n if layer_type == \"conv\":\n if prev_layer == \"fc\":\n raise ValueError(\"Cannot have conv layer after fc layer!\")\n output_node, input_height, input_width, input_channels = \\\n self._build_conv_layer(\n output_node,\n input_height,\n input_width,\n input_channels,\n layer_config[\"filt_dim\"],\n layer_config[\"filt_dim\"],\n layer_config[\"num_filt\"],\n layer_config[\"pool_stride\"],\n layer_config[\"pool_stride\"],\n layer_config[\"pool_size\"],\n layer_name,\n norm=layer_config[\"norm\"],\n pad=layer_config[\"pad\"])\n prev_layer = layer_type\n if layer_config[\"pad\"] == \"SAME\":\n filter_dim //= layer_config[\"pool_stride\"]\n else:\n filter_dim = ((filter_dim - layer_config[\"filt_dim\"]) //\n layer_config[\"pool_stride\"]) + 1\n elif layer_type == \"fc\":\n if layer_index == last_index and only_stream:\n output_node = self._build_fully_conv_layer(\n output_node,\n filter_dim,\n layer_name,\n final_fc_layer=True)\n else:\n output_node = self._build_fully_conv_layer(\n output_node, filter_dim, layer_name)\n prev_layer = layer_type\n # Because fully-convolutional layers at this point in the\n # network have a filter_dim of 1.\n filter_dim = 1\n elif layer_type == \"pc\":\n raise ValueError(\n \"Cannot have pose connected layer in image stream!\")\n elif layer_type == \"fc_merge\":\n raise ValueError(\"Cannot have merge layer in image stream!\")\n else:\n raise ValueError(\n \"Unsupported layer type: {}\".format(layer_type))\n return output_node, -1\n\n def _build_merge_stream(self, input_stream_1, input_stream_2, fan_in_1,\n fan_in_2, drop_rate, layers):\n self._logger.info(\"Building Merge Stream...\")\n\n # First check if first layer is a merge layer.\n if layers[list(layers)[0]][\"type\"] != \"fc_merge\":\n raise ValueError(\n \"First layer in merge stream must be of type fc_merge!\")\n\n last_index = len(layers) - 1\n # Because fully-convolutional layers at this point in the network have\n # a filter_dim of 1.\n filter_dim = 1\n fan_in = -1\n output_node = None # Will be overridden.\n for layer_index, (layer_name,\n layer_config) in enumerate(layers.items()):\n layer_type = layer_config[\"type\"]\n if layer_type == \"conv\":\n raise ValueError(\"Cannot have conv layer in merge stream!\")\n elif layer_type == \"fc\":\n if layer_index == last_index:\n output_node = self._build_fully_conv_layer(\n output_node,\n filter_dim,\n layer_name,\n final_fc_layer=True)\n else:\n output_node = self._build_fully_conv_layer(\n output_node, filter_dim, layer_name)\n elif layer_type == \"pc\":\n raise ValueError(\n \"Cannot have pose connected layer in merge stream!\")\n elif layer_type == \"fc_merge\":\n output_node = self._build_fully_conv_merge_layer(\n input_stream_1, input_stream_2, filter_dim, layer_name)\n else:\n raise ValueError(\n \"Unsupported layer type: {}\".format(layer_type))\n return output_node, fan_in\n" }, { "path": "gqcnn/model/tf/network_tf.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nGQ-CNN network implemented in Tensorflow.\n\nAuthor\n------\nVishal Satish & Jeff Mahler\n\"\"\"\nfrom collections import OrderedDict\nimport errno\nfrom functools import reduce\nimport json\nimport math\nimport operator\nimport os\nimport time\n\nimport numpy as np\nimport tensorflow as tf\nimport tensorflow.contrib.framework as tcf\n\nfrom autolab_core import Logger\nfrom ...utils import (reduce_shape, read_pose_data, pose_dim,\n weight_name_to_layer_name, GripperMode, TrainingMode,\n InputDepthMode, GQCNNFilenames)\n\n\nclass GQCNNWeights(object):\n \"\"\"Helper struct for storing network weights.\"\"\"\n\n def __init__(self):\n self.weights = {}\n\n\nclass GQCNNTF(object):\n \"\"\"GQ-CNN network implemented in Tensorflow.\"\"\"\n\n def __init__(self, gqcnn_config, verbose=True, log_file=None):\n \"\"\"\n Parameters\n ----------\n gqcnn_config : dict\n Python dictionary of model configuration parameters.\n verbose : bool\n Whether or not to log model output to `stdout`.\n log_file : str\n If provided, model output will also be logged to this file.\n \"\"\"\n self._sess = None\n self._graph = tf.Graph()\n\n # Set up logger.\n self._logger = Logger.get_logger(self.__class__.__name__,\n log_file=log_file,\n silence=(not verbose),\n global_log_file=verbose)\n\n self._weights = GQCNNWeights()\n self._parse_config(gqcnn_config)\n\n @staticmethod\n def load(model_dir, verbose=True, log_file=None):\n \"\"\"Instantiate a trained GQ-CNN for fine-tuning or inference.\n\n Parameters\n ----------\n model_dir : str\n Path to trained GQ-CNN model.\n verbose : bool\n Whether or not to log model output to `stdout`.\n log_file : str\n If provided, model output will also be logged to this file.\n\n Returns\n -------\n :obj:`GQCNNTF`\n Initialized GQ-CNN.\n \"\"\"\n config_file = os.path.join(model_dir, GQCNNFilenames.SAVED_CFG)\n with open(config_file) as data_file:\n train_config = json.load(data_file, object_pairs_hook=OrderedDict)\n\n # Support for legacy configs.\n try:\n gqcnn_config = train_config[\"gqcnn\"]\n except KeyError:\n gqcnn_config = train_config[\"gqcnn_config\"]\n\n # Convert old networks to new flexible arch format.\n gqcnn_config[\"debug\"] = 0\n gqcnn_config[\"seed\"] = 0\n # Legacy networks had no angular support.\n gqcnn_config[\"num_angular_bins\"] = 0\n # Legacy networks only supported depth integration through pose\n # stream.\n gqcnn_config[\"input_depth_mode\"] = InputDepthMode.POSE_STREAM\n arch_config = gqcnn_config[\"architecture\"]\n if \"im_stream\" not in arch_config:\n new_arch_config = OrderedDict()\n new_arch_config[\"im_stream\"] = OrderedDict()\n new_arch_config[\"pose_stream\"] = OrderedDict()\n new_arch_config[\"merge_stream\"] = OrderedDict()\n\n layer_name = \"conv1_1\"\n new_arch_config[\"im_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"im_stream\"][layer_name][\"type\"] = \"conv\"\n new_arch_config[\"im_stream\"][layer_name][\"pad\"] = \"SAME\"\n if \"padding\" in arch_config[layer_name]:\n new_arch_config[\"im_stream\"][layer_name][\n \"pad\"] = arch_config[layer_name][\"padding\"]\n\n layer_name = \"conv1_2\"\n new_arch_config[\"im_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"im_stream\"][layer_name][\"type\"] = \"conv\"\n new_arch_config[\"im_stream\"][layer_name][\"pad\"] = \"SAME\"\n if \"padding\" in arch_config[layer_name]:\n new_arch_config[\"im_stream\"][layer_name][\n \"pad\"] = arch_config[layer_name][\"padding\"]\n\n layer_name = \"conv2_1\"\n new_arch_config[\"im_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"im_stream\"][layer_name][\"type\"] = \"conv\"\n new_arch_config[\"im_stream\"][layer_name][\"pad\"] = \"SAME\"\n if \"padding\" in arch_config[layer_name]:\n new_arch_config[\"im_stream\"][layer_name][\n \"pad\"] = arch_config[layer_name][\"padding\"]\n\n layer_name = \"conv2_2\"\n new_arch_config[\"im_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"im_stream\"][layer_name][\"type\"] = \"conv\"\n new_arch_config[\"im_stream\"][layer_name][\"pad\"] = \"SAME\"\n if \"padding\" in arch_config[layer_name]:\n new_arch_config[\"im_stream\"][layer_name][\n \"pad\"] = arch_config[layer_name][\"padding\"]\n\n layer_name = \"conv3_1\"\n if layer_name in arch_config:\n new_arch_config[\"im_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"im_stream\"][layer_name][\"type\"] = \"conv\"\n new_arch_config[\"im_stream\"][layer_name][\"pad\"] = \"SAME\"\n if \"padding\" in arch_config[layer_name]:\n new_arch_config[\"im_stream\"][layer_name][\n \"pad\"] = arch_config[layer_name][\"padding\"]\n\n layer_name = \"conv3_2\"\n if layer_name in arch_config:\n new_arch_config[\"im_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"im_stream\"][layer_name][\"type\"] = \"conv\"\n new_arch_config[\"im_stream\"][layer_name][\"pad\"] = \"SAME\"\n if \"padding\" in arch_config[layer_name]:\n new_arch_config[\"im_stream\"][layer_name][\n \"pad\"] = arch_config[layer_name][\"padding\"]\n\n layer_name = \"fc3\"\n new_arch_config[\"im_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"im_stream\"][layer_name][\"type\"] = \"fc\"\n\n layer_name = \"pc1\"\n new_arch_config[\"pose_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"pose_stream\"][layer_name][\"type\"] = \"pc\"\n\n layer_name = \"pc2\"\n if layer_name in arch_config:\n new_arch_config[\"pose_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"pose_stream\"][layer_name][\"type\"] = \"pc\"\n\n layer_name = \"fc4\"\n new_arch_config[\"merge_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"merge_stream\"][layer_name][\n \"type\"] = \"fc_merge\"\n\n layer_name = \"fc5\"\n new_arch_config[\"merge_stream\"][layer_name] = arch_config[\n layer_name]\n new_arch_config[\"merge_stream\"][layer_name][\"type\"] = \"fc\"\n\n gqcnn_config[\"architecture\"] = new_arch_config\n\n # Initialize weights and Tensorflow network.\n gqcnn = GQCNNTF(gqcnn_config, verbose=verbose, log_file=log_file)\n gqcnn.init_weights_file(\n os.path.join(model_dir, GQCNNFilenames.FINAL_MODEL))\n gqcnn.init_mean_and_std(model_dir)\n training_mode = train_config[\"training_mode\"]\n if training_mode == TrainingMode.CLASSIFICATION:\n gqcnn.initialize_network(add_softmax=True)\n elif training_mode == TrainingMode.REGRESSION:\n gqcnn.initialize_network()\n else:\n raise ValueError(\"Invalid training mode: {}\".format(training_mode))\n return gqcnn\n\n def init_mean_and_std(self, model_dir):\n \"\"\"Loads the means and stds of a trained GQ-CNN to use for data\n normalization during inference.\n\n Parameters\n ----------\n model_dir : str\n Path to trained GQ-CNN model where means and standard deviations\n are stored.\n \"\"\"\n # Load in means and stds.\n if self._input_depth_mode == InputDepthMode.POSE_STREAM:\n try:\n self._im_mean = np.load(\n os.path.join(model_dir, GQCNNFilenames.IM_MEAN))\n self._im_std = np.load(\n os.path.join(model_dir, GQCNNFilenames.IM_STD))\n except IOError as e: # Python 2.6,7/3+ compatibility.\n if e.errno == errno.ENOENT: # File not found.\n # Support for legacy file naming convention.\n self._im_mean = np.load(\n os.path.join(model_dir, GQCNNFilenames.LEG_MEAN))\n self._im_std = np.load(\n os.path.join(model_dir, GQCNNFilenames.LEG_STD))\n else:\n # Some other IOError.\n raise e\n self._pose_mean = np.load(\n os.path.join(model_dir, GQCNNFilenames.POSE_MEAN))\n self._pose_std = np.load(\n os.path.join(model_dir, GQCNNFilenames.POSE_STD))\n\n # Read the certain parts of the pose mean/std that we desire.\n if len(self._pose_mean.shape\n ) > 0 and self._pose_mean.shape[0] != self._pose_dim:\n # Handle multi-dim storage.\n if len(self._pose_mean.shape\n ) > 1 and self._pose_mean.shape[1] == self._pose_dim:\n self._pose_mean = self._pose_mean[0, :]\n self._pose_std = self._pose_std[0, :]\n else:\n self._pose_mean = read_pose_data(self._pose_mean,\n self._gripper_mode)\n self._pose_std = read_pose_data(self._pose_std,\n self._gripper_mode)\n elif self._input_depth_mode == InputDepthMode.SUB:\n self._im_depth_sub_mean = np.load(\n os.path.join(model_dir, GQCNNFilenames.IM_DEPTH_SUB_MEAN))\n self._im_depth_sub_std = np.load(\n os.path.join(model_dir, GQCNNFilenames.IM_DEPTH_SUB_STD))\n elif self._input_depth_mode == InputDepthMode.IM_ONLY:\n self._im_mean = np.load(\n os.path.join(model_dir, GQCNNFilenames.IM_MEAN))\n self._im_std = np.load(\n os.path.join(model_dir, GQCNNFilenames.IM_STD))\n else:\n raise ValueError(\"Unsupported input depth mode: {}\".format(\n self._input_depth_mode))\n\n def set_base_network(self, model_dir):\n \"\"\"Initialize network weights for the base network. Used during\n fine-tuning.\n\n Parameters\n ----------\n model_dir : str\n Path to pre-trained GQ-CNN model.\n \"\"\"\n # Check architecture.\n if \"base_model\" not in self._architecture:\n self._logger.warning(\n \"Architecuture has no base model. The network has not been\"\n \" modified.\")\n return False\n base_model_config = self._architecture[\"base_model\"]\n output_layer = base_model_config[\"output_layer\"]\n\n # Read model.\n ckpt_file = os.path.join(model_dir, GQCNNFilenames.FINAL_MODEL)\n config_file = os.path.join(model_dir, GQCNNFilenames.SAVED_ARCH)\n base_arch = json.load(open(config_file, \"r\"),\n object_pairs_hook=OrderedDict)\n\n # Read base layer names.\n self._base_layer_names = []\n found_base_layer = False\n use_legacy = not (\"im_stream\" in base_arch)\n if use_legacy:\n layer_iter = iter(base_arch)\n while not found_base_layer:\n layer_name = next(layer_iter)\n self._base_layer_names.append(layer_name)\n if layer_name == output_layer:\n found_base_layer = True\n else:\n stream_iter = iter(base_arch)\n while not found_base_layer:\n stream_name = next(stream_iter)\n stream_arch = base_arch[stream_name]\n layer_iter = iter(stream_arch)\n stop = False\n while not found_base_layer and not stop:\n try:\n layer_name = next(layer_iter)\n self._base_layer_names.append(layer_name)\n if layer_name == output_layer:\n found_base_layer = True\n except StopIteration:\n stop = True\n\n with self._graph.as_default():\n # Create new tf checkpoint reader.\n reader = tf.train.NewCheckpointReader(ckpt_file)\n\n # Create empty weights object.\n self._weights = GQCNNWeights()\n\n # Read/generate weight/bias variable names.\n ckpt_vars = tcf.list_variables(ckpt_file)\n full_var_names = []\n short_names = []\n for variable, shape in ckpt_vars:\n full_var_names.append(variable)\n short_names.append(variable.split(\"/\")[-1])\n\n # Load variables.\n for full_var_name, short_name in zip(full_var_names, short_names):\n # Check valid weights.\n layer_name = weight_name_to_layer_name(short_name)\n\n # Add weights.\n if layer_name in self._base_layer_names:\n self._weights.weights[short_name] = tf.Variable(\n reader.get_tensor(full_var_name), name=full_var_name)\n\n def init_weights_file(self, ckpt_file):\n \"\"\"Load trained GQ-CNN weights.\n\n Parameters\n ----------\n ckpt_file : str\n Tensorflow checkpoint file from which to load model weights.\n \"\"\"\n with self._graph.as_default():\n # Create new tf checkpoint reader.\n reader = tf.train.NewCheckpointReader(ckpt_file)\n\n # Create empty weight object.\n self._weights = GQCNNWeights()\n\n # Read/generate weight/bias variable names.\n ckpt_vars = tcf.list_variables(ckpt_file)\n full_var_names = []\n short_names = []\n for variable, shape in ckpt_vars:\n full_var_names.append(variable)\n short_names.append(variable.split(\"/\")[-1])\n\n # Load variables.\n for full_var_name, short_name in zip(full_var_names, short_names):\n self._weights.weights[short_name] = tf.Variable(\n reader.get_tensor(full_var_name), name=full_var_name)\n\n def _parse_config(self, gqcnn_config):\n \"\"\"Parse configuration file.\n\n Parameters\n ----------\n gqcnn_config : dict\n Python dictionary of model configuration parameters.\n \"\"\"\n\n # Parse GQ-CNN config.\n # Load tensor params.\n self._batch_size = gqcnn_config[\"batch_size\"]\n self._train_im_height = gqcnn_config[\"im_height\"]\n self._train_im_width = gqcnn_config[\"im_width\"]\n self._im_height = self._train_im_height\n self._im_width = self._train_im_width\n self._num_channels = gqcnn_config[\"im_channels\"]\n try:\n self._gripper_mode = gqcnn_config[\"gripper_mode\"]\n except KeyError:\n # Legacy support.\n self._input_data_mode = gqcnn_config[\"input_data_mode\"]\n if self._input_data_mode == \"tf_image\":\n self._gripper_mode = GripperMode.LEGACY_PARALLEL_JAW\n elif self._input_data_mode == \"tf_image_suction\":\n self._gripper_mode = GripperMode.LEGACY_SUCTION\n elif self._input_data_mode == \"suction\":\n self._gripper_mode = GripperMode.SUCTION\n elif self._input_data_mode == \"multi_suction\":\n self._gripper_mode = GripperMode.MULTI_SUCTION\n elif self._input_data_mode == \"parallel_jaw\":\n self._gripper_mode = GripperMode.PARALLEL_JAW\n else:\n raise ValueError(\n \"Legacy input data mode: {} not supported!\".format(\n self._input_data_mode))\n self._logger.warning(\"Could not read gripper mode. Attempting\"\n \" legacy conversion to: {}\".format(\n self._gripper_mode))\n\n # Setup gripper pose dimension depending on gripper mode.\n self._pose_dim = pose_dim(self._gripper_mode)\n\n # Load architecture.\n self._architecture = gqcnn_config[\"architecture\"]\n\n # Get input depth mode.\n self._input_depth_mode = InputDepthMode.POSE_STREAM # Legacy support.\n if \"input_depth_mode\" in gqcnn_config:\n self._input_depth_mode = gqcnn_config[\"input_depth_mode\"]\n\n # Load network local response normalization layer constants.\n self._normalization_radius = gqcnn_config[\"radius\"]\n self._normalization_alpha = gqcnn_config[\"alpha\"]\n self._normalization_beta = gqcnn_config[\"beta\"]\n self._normalization_bias = gqcnn_config[\"bias\"]\n\n # Get ReLU coefficient.\n self._relu_coeff = 0.0 # Legacy support.\n if \"relu_coeff\" in gqcnn_config:\n self._relu_coeff = gqcnn_config[\"relu_coeff\"]\n\n # Debugging.\n self._debug = gqcnn_config[\"debug\"]\n self._rand_seed = gqcnn_config[\"seed\"]\n\n # Initialize means and standard deviations to be 0 and 1, respectively.\n if self._input_depth_mode == InputDepthMode.POSE_STREAM:\n self._im_mean = 0\n self._im_std = 1\n self._pose_mean = np.zeros(self._pose_dim)\n self._pose_std = np.ones(self._pose_dim)\n elif self._input_depth_mode == InputDepthMode.SUB:\n self._im_depth_sub_mean = 0\n self._im_depth_sub_std = 1\n elif self._input_depth_mode == InputDepthMode.IM_ONLY:\n self._im_mean = 0\n self._im_std = 1\n\n # Get number of angular bins.\n self._angular_bins = 0 # Legacy support.\n if \"angular_bins\" in gqcnn_config:\n self._angular_bins = gqcnn_config[\"angular_bins\"]\n\n # Get max angle.\n self._max_angle = np.pi\n if \"max_angle\" in gqcnn_config:\n self._max_angle = np.deg2rad(gqcnn_config[\"max_angle\"])\n\n # If using angular bins, make sure output size of final fully connected\n # layer is 2x number of angular bins (because of failure/success probs\n # for each bin).\n if self._angular_bins > 0:\n final_out_size = list(\n list(self._architecture.values())[-1].values())[-1][\"out_size\"]\n ang_mismatch_msg = (\"When predicting angular outputs, output\"\n \" size of final fully connected layer must\"\n \" be 2x number of angular bins.\")\n assert final_out_size == 2 * self._angular_bins, ang_mismatch_msg\n\n # Intermediate network feature handles.\n self._feature_tensors = {}\n\n # Base layer names for fine-tuning.\n self._base_layer_names = []\n\n def initialize_network(self,\n train_im_node=None,\n train_pose_node=None,\n add_softmax=False,\n add_sigmoid=False):\n \"\"\"Set up input placeholders and build network.\n\n Parameters\n ----------\n train_im_node : :obj:`tf.placeholder`\n Images for training.\n train_pose_node : :obj:`tf.placeholder`\n Poses for training.\n add_softmax : bool\n Whether or not to add a softmax layer to output of network.\n add_sigmoid : bool\n Whether or not to add a sigmoid layer to output of network.\n \"\"\"\n with self._graph.as_default():\n # Set TF random seed if debugging.\n if self._debug:\n tf.set_random_seed(self._rand_seed)\n\n # Setup input placeholders.\n if train_im_node is not None:\n # Training.\n self._input_im_node = tf.placeholder_with_default(\n train_im_node, (None, self._im_height, self._im_width,\n self._num_channels))\n self._input_pose_node = tf.placeholder_with_default(\n train_pose_node, (None, self._pose_dim))\n else:\n # Inference only using GQ-CNN instantiated from `GQCNNTF.load`.\n self._input_im_node = tf.placeholder(\n tf.float32, (self._batch_size, self._im_height,\n self._im_width, self._num_channels))\n self._input_pose_node = tf.placeholder(\n tf.float32, (self._batch_size, self._pose_dim))\n self._input_drop_rate_node = tf.placeholder_with_default(\n tf.constant(0.0), ())\n\n # Build network.\n self._output_tensor = self._build_network(\n self._input_im_node, self._input_pose_node,\n self._input_drop_rate_node)\n\n # Add softmax function to output of network (this is optional\n # because 1) we might be doing regression or 2) we are training and\n # Tensorflow has an optimized cross-entropy loss with the softmax\n # already built-in).\n if add_softmax:\n self.add_softmax_to_output()\n # Add sigmoid function to output of network (for weighted\n # cross-entropy loss).\n if add_sigmoid:\n self.add_sigmoid_to_output()\n\n # Create feed tensors for prediction.\n self._input_im_arr = np.zeros((self._batch_size, self._im_height,\n self._im_width, self._num_channels))\n self._input_pose_arr = np.zeros((self._batch_size, self._pose_dim))\n\n def open_session(self):\n \"\"\"Open Tensorflow session.\"\"\"\n if self._sess is not None:\n self._logger.warning(\"Found already initialized TF Session...\")\n return self._sess\n self._logger.info(\"Initializing TF Session...\")\n with self._graph.as_default():\n init = tf.global_variables_initializer()\n self.tf_config = tf.ConfigProto()\n # Allow Tensorflow gpu growth so Tensorflow does not lock-up all\n # GPU memory.\n self.tf_config.gpu_options.allow_growth = True\n self._sess = tf.Session(graph=self._graph, config=self.tf_config)\n self._sess.run(init)\n return self._sess\n\n def close_session(self):\n \"\"\"Close Tensorflow session.\"\"\"\n if self._sess is None:\n self._logger.warning(\"No TF Session to close...\")\n return\n self._logger.info(\"Closing TF Session...\")\n with self._graph.as_default():\n self._sess.close()\n self._sess = None\n\n def __del__(self):\n \"\"\"Destructor that basically just makes sure the Tensorflow session\n has been closed.\"\"\"\n if self._sess is not None:\n self.close_session()\n\n @property\n def input_depth_mode(self):\n return self._input_depth_mode\n\n @property\n def batch_size(self):\n return self._batch_size\n\n @property\n def im_height(self):\n return self._im_height\n\n @property\n def im_width(self):\n return self._im_width\n\n @property\n def num_channels(self):\n return self._num_channels\n\n @property\n def pose_dim(self):\n return self._pose_dim\n\n @property\n def gripper_mode(self):\n return self._gripper_mode\n\n @property\n def input_im_node(self):\n return self._input_im_node\n\n @property\n def input_pose_node(self):\n return self._input_pose_node\n\n @property\n def input_drop_rate_node(self):\n return self._input_drop_rate_node\n\n @property\n def output(self):\n return self._output_tensor\n\n @property\n def weights(self):\n return self._weights.weights\n\n @property\n def tf_graph(self):\n return self._graph\n\n @property\n def sess(self):\n return self._sess\n\n @property\n def angular_bins(self):\n return self._angular_bins\n\n @property\n def max_angle(self):\n return self._max_angle\n\n @property\n def stride(self):\n return reduce(operator.mul, [\n layer[\"pool_stride\"]\n for layer in self._architecture[\"im_stream\"].values()\n if layer[\"type\"] == \"conv\"\n ])\n\n @property\n def filters(self):\n \"\"\"Evaluate the filters of the first convolution layer.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Filters (weights) from first convolution layer of the network.\n \"\"\"\n close_sess = False\n if self._sess is None:\n close_sess = True\n self.open_session()\n\n first_layer_name = list(self._architecture[\"im_stream\"])[0]\n try:\n filters = self._sess.run(\n self._weights.weights[\"{}_weights\".format(first_layer_name)])\n except KeyError:\n # Legacy support.\n filters = self._sess.run(\n self._weights.weights[\"{}W\".format(first_layer_name)])\n\n if close_sess:\n self.close_session()\n return filters\n\n def set_batch_size(self, batch_size):\n \"\"\"Update the batch size to be used for during inference.\n\n Parameters\n ----------\n batch_size : int\n The new batch size.\n \"\"\"\n self._batch_size = batch_size\n\n def set_im_mean(self, im_mean):\n \"\"\"Update image mean to be used for normalization during inference.\n\n Parameters\n ----------\n im_mean : float\n The new image mean.\n \"\"\"\n self._im_mean = im_mean\n\n def get_im_mean(self):\n \"\"\"Get the current image mean used for normalization during inference.\n\n Returns\n -------\n : float\n The image mean.\n \"\"\"\n return self.im_mean\n\n def set_im_std(self, im_std):\n \"\"\"Update image standard deviation to be used for normalization during\n inference.\n\n Parameters\n ----------\n im_std : float\n The new image standard deviation.\n \"\"\"\n self._im_std = im_std\n\n def get_im_std(self):\n \"\"\"Get the current image standard deviation to be used for\n normalization during inference.\n\n Returns\n -------\n : float\n The image standard deviation.\n \"\"\"\n return self.im_std\n\n def set_pose_mean(self, pose_mean):\n \"\"\"Update pose mean to be used for normalization during inference.\n\n Parameters\n ----------\n pose_mean : :obj:`numpy.ndarray`\n The new pose mean.\n \"\"\"\n self._pose_mean = pose_mean\n\n def get_pose_mean(self):\n \"\"\"Get the current pose mean to be used for normalization during\n inference.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n The pose mean.\n \"\"\"\n return self._pose_mean\n\n def set_pose_std(self, pose_std):\n \"\"\"Update pose standard deviation to be used for normalization during\n inference.\n\n Parameters\n ----------\n pose_std : :obj:`numpy.ndarray`\n The new pose standard deviation.\n \"\"\"\n self._pose_std = pose_std\n\n def get_pose_std(self):\n \"\"\"Get the current pose standard deviation to be used for normalization\n during inference.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n The pose standard deviation.\n \"\"\"\n return self._pose_std\n\n def set_im_depth_sub_mean(self, im_depth_sub_mean):\n \"\"\"Update mean of subtracted image and gripper depth to be used for\n normalization during inference.\n\n Parameters\n ----------\n im_depth_sub_mean : float\n The new mean of subtracted image and gripper depth.\n \"\"\"\n self._im_depth_sub_mean = im_depth_sub_mean\n\n def set_im_depth_sub_std(self, im_depth_sub_std):\n \"\"\"Update standard deviation of subtracted image and gripper depth to\n be used for normalization during inference.\n\n Parameters\n ----------\n im_depth_sub_std : float\n The standard deviation of subtracted image and gripper depth.\n \"\"\"\n self._im_depth_sub_std = im_depth_sub_std\n\n def add_softmax_to_output(self):\n \"\"\"Adds softmax to output of network.\"\"\"\n with tf.name_scope(\"softmax\"):\n if self._angular_bins > 0:\n self._logger.info(\"Building Pair-wise Softmax Layer...\")\n binwise_split_output = tf.split(self._output_tensor,\n self._angular_bins,\n axis=-1)\n binwise_split_output_soft = [\n tf.nn.softmax(s) for s in binwise_split_output\n ]\n self._output_tensor = tf.concat(binwise_split_output_soft, -1)\n else:\n self._logger.info(\"Building Softmax Layer...\")\n self._output_tensor = tf.nn.softmax(self._output_tensor)\n\n def add_sigmoid_to_output(self):\n \"\"\"Adds sigmoid to output of network.\"\"\"\n with tf.name_scope(\"sigmoid\"):\n self._logger.info(\"Building Sigmoid Layer...\")\n self._output_tensor = tf.nn.sigmoid(self._output_tensor)\n\n def update_batch_size(self, batch_size):\n \"\"\"Update the inference batch size.\n\n Parameters\n ----------\n batch_size : float\n The new batch size.\n \"\"\"\n self._batch_size = batch_size\n\n def _predict(self, image_arr, pose_arr, verbose=False):\n \"\"\"Query predictions from the network.\n\n Parameters\n ----------\n image_arr : :obj:`numpy.ndarray`\n Input images.\n pose_arr : :obj:`numpy.ndarray`\n Input gripper poses.\n verbose : bool\n Whether or not to log progress to `stdout`, useful to turn off\n during training.\n \"\"\"\n # Get prediction start time.\n start_time = time.time()\n\n if verbose:\n self._logger.info(\"Predicting...\")\n\n # Setup for prediction.\n num_batches = math.ceil(image_arr.shape[0] / self._batch_size)\n num_images = image_arr.shape[0]\n num_poses = pose_arr.shape[0]\n\n output_arr = None\n if num_images != num_poses:\n raise ValueError(\"Must provide same number of images as poses!\")\n\n # Predict in batches.\n with self._graph.as_default():\n if self._sess is None:\n raise RuntimeError(\n \"No TF Session open. Please call open_session() first.\")\n i = 0\n batch_idx = 0\n while i < num_images:\n if verbose:\n self._logger.info(\"Predicting batch {} of {}...\".format(\n batch_idx, num_batches))\n batch_idx += 1\n dim = min(self._batch_size, num_images - i)\n cur_ind = i\n end_ind = cur_ind + dim\n\n if self._input_depth_mode == InputDepthMode.POSE_STREAM:\n self._input_im_arr[:dim,\n ...] = (image_arr[cur_ind:end_ind, ...]\n - self._im_mean) / self._im_std\n self._input_pose_arr[:dim, :] = (\n pose_arr[cur_ind:end_ind, :] -\n self._pose_mean) / self._pose_std\n elif self._input_depth_mode == InputDepthMode.SUB:\n self._input_im_arr[:dim, ...] = image_arr[cur_ind:end_ind,\n ...]\n self._input_pose_arr[:dim, :] = pose_arr[\n cur_ind:end_ind, :]\n elif self._input_depth_mode == InputDepthMode.IM_ONLY:\n self._input_im_arr[:dim,\n ...] = (image_arr[cur_ind:end_ind, ...]\n - self._im_mean) / self._im_std\n\n gqcnn_output = self._sess.run(\n self._output_tensor,\n feed_dict={\n self._input_im_node: self._input_im_arr,\n self._input_pose_node: self._input_pose_arr\n })\n\n # Allocate output tensor.\n if output_arr is None:\n output_arr = np.zeros([num_images] +\n list(gqcnn_output.shape[1:]))\n\n output_arr[cur_ind:end_ind, :] = gqcnn_output[:dim, :]\n i = end_ind\n\n # Get total prediction time.\n pred_time = time.time() - start_time\n if verbose:\n self._logger.info(\"Prediction took {} seconds.\".format(pred_time))\n\n return output_arr\n\n def predict(self, image_arr, pose_arr, verbose=False):\n \"\"\"Predict the probability of grasp success given a depth image and\n gripper pose.\n\n Parameters\n ----------\n image_arr : :obj:`numpy ndarray`\n 4D tensor of depth images.\n pose_arr : :obj:`numpy ndarray`\n Tensor of gripper poses.\n verbose : bool\n Whether or not to log progress to stdout, useful to turn off during\n training.\n \"\"\"\n return self._predict(image_arr, pose_arr, verbose=verbose)\n\n def featurize(self,\n image_arr,\n pose_arr=None,\n feature_layer=\"conv1_1\",\n verbose=False):\n \"\"\"Featurize a set of inputs.\n\n Parameters\n ----------\n image_arr : :obj:`numpy ndarray`\n 4D tensor of depth images.\n pose_arr : :obj:`numpy ndarray`\n Optional tensor of gripper poses.\n feature_layer : str\n The network layer to featurize.\n verbose : bool\n Whether or not to log progress to `stdout`.\n \"\"\"\n # Get featurization start time.\n start_time = time.time()\n\n if verbose:\n self._logger.info(\"Featurizing...\")\n\n if feature_layer not in self._feature_tensors:\n raise ValueError(\n \"Feature layer: {} not recognized.\".format(feature_layer))\n\n # Setup for featurization.\n num_images = image_arr.shape[0]\n if pose_arr is not None:\n num_poses = pose_arr.shape[0]\n if num_images != num_poses:\n raise ValueError(\n \"Must provide same number of images as poses!\")\n output_arr = None\n\n # Featurize in batches.\n with self._graph.as_default():\n if self._sess is None:\n raise RuntimeError(\n \"No TF Session open. Please call open_session() first.\")\n\n i = 0\n while i < num_images:\n if verbose:\n self._logger.info(\"Featurizing {} of {}...\".format(\n i, num_images))\n dim = min(self._batch_size, num_images - i)\n cur_ind = i\n end_ind = cur_ind + dim\n self._input_im_arr[:dim, :, :, :] = (\n image_arr[cur_ind:end_ind, :, :, :] -\n self._im_mean) / self._im_std\n if pose_arr is not None:\n self._input_pose_arr[:dim, :] = (\n pose_arr[cur_ind:end_ind, :] -\n self._pose_mean) / self._pose_std\n\n if pose_arr is not None:\n gqcnn_output = self._sess.run(\n self._feature_tensors[feature_layer],\n feed_dict={\n self._input_im_node: self._input_im_arr,\n self._input_pose_node: self._input_pose_arr\n })\n else:\n gqcnn_output = self._sess.run(\n self._feature_tensors[feature_layer],\n feed_dict={self._input_im_node: self._input_im_arr})\n\n if output_arr is None:\n output_arr = np.zeros([num_images] +\n list(gqcnn_output.shape[1:]))\n output_arr[cur_ind:end_ind, :] = gqcnn_output[:dim, :]\n\n i = end_ind\n\n if verbose:\n self._logger.info(\n \"Featurization took {} seconds\".format(time.time() -\n start_time))\n\n # Truncate extraneous values off of end of `output_arr`.\n # TODO(vsatish): This isn't needed, right?\n output_arr = output_arr[:num_images]\n return output_arr\n\n def _leaky_relu(self, x, alpha=.1):\n return tf.maximum(alpha * x, x)\n\n def _build_conv_layer(self,\n input_node,\n input_height,\n input_width,\n input_channels,\n filter_h,\n filter_w,\n num_filt,\n pool_stride_h,\n pool_stride_w,\n pool_size,\n name,\n norm=False,\n pad=\"SAME\"):\n self._logger.info(\"Building convolutional layer: {}...\".format(name))\n with tf.name_scope(name):\n # Initialize weights.\n if \"{}_weights\".format(name) in self._weights.weights:\n convW = self._weights.weights[\"{}_weights\".format(name)]\n convb = self._weights.weights[\"{}_bias\".format(name)]\n elif \"{}W\".format(\n name) in self._weights.weights: # Legacy support.\n self._logger.info(\n \"Using old format for layer {}.\".format(name))\n convW = self._weights.weights[\"{}W\".format(name)]\n convb = self._weights.weights[\"{}b\".format(name)]\n else:\n self._logger.info(\"Reinitializing layer {}.\".format(name))\n convW_shape = [filter_h, filter_w, input_channels, num_filt]\n\n fan_in = filter_h * filter_w * input_channels\n std = np.sqrt(2 / fan_in)\n convW = tf.Variable(tf.truncated_normal(convW_shape,\n stddev=std),\n name=\"{}_weights\".format(name))\n convb = tf.Variable(tf.truncated_normal([num_filt],\n stddev=std),\n name=\"{}_bias\".format(name))\n\n self._weights.weights[\"{}_weights\".format(name)] = convW\n self._weights.weights[\"{}_bias\".format(name)] = convb\n\n if pad == \"SAME\":\n out_height = input_height // pool_stride_h\n out_width = input_width // pool_stride_w\n else:\n out_height = math.ceil(\n (input_height - filter_h + 1) / pool_stride_h)\n out_width = math.ceil(\n (input_width - filter_w + 1) / pool_stride_w)\n out_channels = num_filt\n\n # Build layer.\n convh = tf.nn.conv2d(\n input_node, convW, strides=[1, 1, 1, 1], padding=pad) + convb\n convh = self._leaky_relu(convh, alpha=self._relu_coeff)\n\n if norm:\n convh = tf.nn.local_response_normalization(\n convh,\n depth_radius=self._normalization_radius,\n alpha=self._normalization_alpha,\n beta=self._normalization_beta,\n bias=self._normalization_bias)\n pool = tf.nn.max_pool(convh,\n ksize=[1, pool_size, pool_size, 1],\n strides=[1, pool_stride_h, pool_stride_w, 1],\n padding=\"SAME\")\n\n # Add output to feature dict.\n self._feature_tensors[name] = pool\n\n return pool, out_height, out_width, out_channels\n\n def _build_fc_layer(self,\n input_node,\n fan_in,\n out_size,\n name,\n input_is_multi,\n drop_rate,\n final_fc_layer=False):\n self._logger.info(\"Building fully connected layer: {}...\".format(name))\n\n # Initialize weights.\n if \"{}_weights\".format(name) in self._weights.weights:\n fcW = self._weights.weights[\"{}_weights\".format(name)]\n fcb = self._weights.weights[\"{}_bias\".format(name)]\n elif \"{}W\".format(name) in self._weights.weights: # Legacy support.\n self._logger.info(\"Using old format for layer {}.\".format(name))\n fcW = self._weights.weights[\"{}W\".format(name)]\n fcb = self._weights.weights[\"{}b\".format(name)]\n else:\n self._logger.info(\"Reinitializing layer {}.\".format(name))\n std = np.sqrt(2 / fan_in)\n fcW = tf.Variable(tf.truncated_normal([fan_in, out_size],\n stddev=std),\n name=\"{}_weights\".format(name))\n if final_fc_layer:\n fcb = tf.Variable(tf.constant(0.0, shape=[out_size]),\n name=\"{}_bias\".format(name))\n else:\n fcb = tf.Variable(tf.truncated_normal([out_size], stddev=std),\n name=\"{}_bias\".format(name))\n\n self._weights.weights[\"{}_weights\".format(name)] = fcW\n self._weights.weights[\"{}_bias\".format(name)] = fcb\n\n # Build layer.\n if input_is_multi:\n reduced_dim1 = reduce_shape(input_node.get_shape())\n input_node = tf.reshape(input_node, [-1, reduced_dim1])\n if final_fc_layer:\n fc = tf.matmul(input_node, fcW) + fcb\n else:\n fc = self._leaky_relu(tf.matmul(input_node, fcW) + fcb,\n alpha=self._relu_coeff)\n\n fc = tf.nn.dropout(fc, 1 - drop_rate)\n\n # Add output to feature dict.\n self._feature_tensors[name] = fc\n\n return fc, out_size\n\n # TODO(vsatish): This really doesn't need to it's own layer type...it does\n # the same thing as `_build_fc_layer`.\n def _build_pc_layer(self, input_node, fan_in, out_size, name):\n self._logger.info(\n \"Building Fully Connected Pose Layer: {}...\".format(name))\n\n # Initialize weights.\n if \"{}_weights\".format(name) in self._weights.weights:\n pcW = self._weights.weights[\"{}_weights\".format(name)]\n pcb = self._weights.weights[\"{}_bias\".format(name)]\n elif \"{}W\".format(name) in self._weights.weights: # Legacy support.\n self._logger.info(\"Using old format for layer {}\".format(name))\n pcW = self._weights.weights[\"{}W\".format(name)]\n pcb = self._weights.weights[\"{}b\".format(name)]\n else:\n self._logger.info(\"Reinitializing layer {}\".format(name))\n std = np.sqrt(2 / fan_in)\n pcW = tf.Variable(tf.truncated_normal([fan_in, out_size],\n stddev=std),\n name=\"{}_weights\".format(name))\n pcb = tf.Variable(tf.truncated_normal([out_size], stddev=std),\n name=\"{}_bias\".format(name))\n\n self._weights.weights[\"{}_weights\".format(name)] = pcW\n self._weights.weights[\"{}_bias\".format(name)] = pcb\n\n # Build layer.\n pc = self._leaky_relu(tf.matmul(input_node, pcW) + pcb,\n alpha=self._relu_coeff)\n\n # Add output to feature dict.\n self._feature_tensors[name] = pc\n\n return pc, out_size\n\n def _build_fc_merge(self, input_fc_node_1, input_fc_node_2, fan_in_1,\n fan_in_2, out_size, drop_rate, name):\n self._logger.info(\"Building Merge Layer: {}...\".format(name))\n\n # Initialize weights.\n if \"{}_input_1_weights\".format(name) in self._weights.weights:\n input1W = self._weights.weights[\"{}_input_1_weights\".format(name)]\n input2W = self._weights.weights[\"{}_input_2_weights\".format(name)]\n fcb = self._weights.weights[\"{}_bias\".format(name)]\n elif \"{}W_im\".format(name) in self._weights.weights: # Legacy support.\n self._logger.info(\"Using old format for layer {}.\".format(name))\n input1W = self._weights.weights[\"{}W_im\".format(name)]\n input2W = self._weights.weights[\"{}W_pose\".format(name)]\n fcb = self._weights.weights[\"{}b\".format(name)]\n else:\n self._logger.info(\"Reinitializing layer {}.\".format(name))\n std = np.sqrt(2 / (fan_in_1 + fan_in_2))\n input1W = tf.Variable(tf.truncated_normal([fan_in_1, out_size],\n stddev=std),\n name=\"{}_input_1_weights\".format(name))\n input2W = tf.Variable(tf.truncated_normal([fan_in_2, out_size],\n stddev=std),\n name=\"{}_input_2_weights\".format(name))\n fcb = tf.Variable(tf.truncated_normal([out_size], stddev=std),\n name=\"{}_bias\".format(name))\n\n self._weights.weights[\"{}_input_1_weights\".format(name)] = input1W\n self._weights.weights[\"{}_input_2_weights\".format(name)] = input2W\n self._weights.weights[\"{}_bias\".format(name)] = fcb\n\n # Build layer.\n fc = self._leaky_relu(tf.matmul(input_fc_node_1, input1W) +\n tf.matmul(input_fc_node_2, input2W) + fcb,\n alpha=self._relu_coeff)\n fc = tf.nn.dropout(fc, 1 - drop_rate)\n\n # Add output to feature dict.\n self._feature_tensors[name] = fc\n\n return fc, out_size\n\n def _build_im_stream(self,\n input_node,\n input_pose_node,\n input_height,\n input_width,\n input_channels,\n drop_rate,\n layers,\n only_stream=False):\n self._logger.info(\"Building Image Stream...\")\n\n if self._input_depth_mode == InputDepthMode.SUB:\n sub_mean = tf.constant(self._im_depth_sub_mean, dtype=tf.float32)\n sub_std = tf.constant(self._im_depth_sub_std, dtype=tf.float32)\n sub_im = tf.subtract(\n input_node,\n tf.tile(\n tf.reshape(input_pose_node, tf.constant((-1, 1, 1, 1))),\n tf.constant((1, input_height, input_width, 1))))\n norm_sub_im = tf.div(tf.subtract(sub_im, sub_mean), sub_std)\n input_node = norm_sub_im\n\n output_node = input_node\n prev_layer = \"start\" # Dummy placeholder.\n last_index = len(layers) - 1\n for layer_index, (layer_name,\n layer_config) in enumerate(layers.items()):\n layer_type = layer_config[\"type\"]\n if layer_type == \"conv\":\n if prev_layer == \"fc\":\n raise ValueError(\"Cannot have conv layer after fc layer!\")\n output_node, input_height, input_width, input_channels = \\\n self._build_conv_layer(\n output_node,\n input_height,\n input_width,\n input_channels,\n layer_config[\"filt_dim\"],\n layer_config[\"filt_dim\"],\n layer_config[\"num_filt\"],\n layer_config[\"pool_stride\"],\n layer_config[\"pool_stride\"],\n layer_config[\"pool_size\"],\n layer_name,\n norm=layer_config[\"norm\"],\n pad=layer_config[\"pad\"])\n prev_layer = layer_type\n elif layer_type == \"fc\":\n if layer_config[\"out_size\"] == 0:\n continue\n prev_layer_is_conv = False\n if prev_layer == \"conv\":\n prev_layer_is_conv = True\n fan_in = input_height * input_width * input_channels\n if layer_index == last_index and only_stream:\n output_node, fan_in = self._build_fc_layer(\n output_node,\n fan_in,\n layer_config[\"out_size\"],\n layer_name,\n prev_layer_is_conv,\n drop_rate,\n final_fc_layer=True)\n else:\n output_node, fan_in = self._build_fc_layer(\n output_node, fan_in, layer_config[\"out_size\"],\n layer_name, prev_layer_is_conv, drop_rate)\n prev_layer = layer_type\n elif layer_type == \"pc\":\n raise ValueError(\n \"Cannot have pose connected layer in image stream!\")\n elif layer_type == \"fc_merge\":\n raise ValueError(\"Cannot have merge layer in image stream!\")\n else:\n raise ValueError(\n \"Unsupported layer type: {}\".format(layer_type))\n return output_node, fan_in\n\n def _build_pose_stream(self, input_node, fan_in, layers):\n self._logger.info(\"Building Pose Stream...\")\n output_node = input_node\n for layer_name, layer_config in layers.items():\n layer_type = layer_config[\"type\"]\n if layer_type == \"conv\":\n raise ValueError(\"Cannot have conv layer in pose stream\")\n elif layer_type == \"fc\":\n raise ValueError(\n \"Cannot have fully connected layer in pose stream\")\n elif layer_type == \"pc\":\n if layer_config[\"out_size\"] == 0:\n continue\n output_node, fan_in = self._build_pc_layer(\n output_node, fan_in, layer_config[\"out_size\"], layer_name)\n elif layer_type == \"fc_merge\":\n raise ValueError(\"Cannot have merge layer in pose stream\")\n else:\n raise ValueError(\n \"Unsupported layer type: {}\".format(layer_type))\n\n return output_node, fan_in\n\n def _build_merge_stream(self, input_stream_1, input_stream_2, fan_in_1,\n fan_in_2, drop_rate, layers):\n self._logger.info(\"Building Merge Stream...\")\n\n # First check if first layer is a merge layer.\n # TODO(vsatish): Can't we just get the first value because it's\n # ordered?\n if layers[list(layers)[0]][\"type\"] != \"fc_merge\":\n raise ValueError(\n \"First layer in merge stream must be a fc_merge layer!\")\n\n last_index = len(layers) - 1\n fan_in = -1\n output_node = None # Will be overridden.\n for layer_index, (layer_name,\n layer_config) in enumerate(layers.items()):\n layer_type = layer_config[\"type\"]\n if layer_type == \"conv\":\n raise ValueError(\"Cannot have conv layer in merge stream!\")\n elif layer_type == \"fc\":\n if layer_config[\"out_size\"] == 0:\n continue\n if layer_index == last_index:\n output_node, fan_in = self._build_fc_layer(\n output_node,\n fan_in,\n layer_config[\"out_size\"],\n layer_name,\n False,\n drop_rate,\n final_fc_layer=True)\n else:\n output_node, fan_in = self._build_fc_layer(\n output_node, fan_in, layer_config[\"out_size\"],\n layer_name, False, drop_rate)\n elif layer_type == \"pc\":\n raise ValueError(\n \"Cannot have pose connected layer in merge stream!\")\n elif layer_type == \"fc_merge\":\n if layer_config[\"out_size\"] == 0:\n continue\n output_node, fan_in = self._build_fc_merge(\n input_stream_1, input_stream_2, fan_in_1, fan_in_2,\n layer_config[\"out_size\"], drop_rate, layer_name)\n else:\n raise ValueError(\n \"Unsupported layer type: {}\".format(layer_type))\n return output_node, fan_in\n\n def _build_network(self, input_im_node, input_pose_node,\n input_drop_rate_node):\n \"\"\"Build GQ-CNN.\n\n Parameters\n ----------\n input_im_node :obj:`tf.placeholder`\n Image placeholder.\n input_pose_node :obj:`tf.placeholder`\n Gripper pose placeholder.\n input_drop_rate_node :obj:`tf.placeholder`\n Drop rate placeholder.\n\n Returns\n -------\n :obj:`tf.Tensor`\n Tensor output of network.\n \"\"\"\n self._logger.info(\"Building Network...\")\n if self._input_depth_mode == InputDepthMode.POSE_STREAM:\n missing_stream_msg = (\"When using input depth mode\"\n \" 'pose_stream', both pose stream and\"\n \" merge stream must be present!\")\n assert \"pose_stream\" in self._architecture and \\\n \"merge_stream\" in self._architecture, missing_stream_msg\n with tf.name_scope(\"im_stream\"):\n output_im_stream, fan_out_im = self._build_im_stream(\n input_im_node, input_pose_node, self._im_height,\n self._im_width, self._num_channels, input_drop_rate_node,\n self._architecture[\"im_stream\"])\n with tf.name_scope(\"pose_stream\"):\n output_pose_stream, fan_out_pose = self._build_pose_stream(\n input_pose_node, self._pose_dim,\n self._architecture[\"pose_stream\"])\n with tf.name_scope(\"merge_stream\"):\n return self._build_merge_stream(\n output_im_stream, output_pose_stream, fan_out_im,\n fan_out_pose, input_drop_rate_node,\n self._architecture[\"merge_stream\"])[0]\n elif self._input_depth_mode == InputDepthMode.SUB or \\\n self._input_depth_mode == InputDepthMode.IM_ONLY:\n extraneous_stream_msg = (\"When using input depth mode '{}', only\"\n \" im stream is allowed!\")\n assert not (\"pose_stream\" in self._architecture or \"merge_stream\"\n in self._architecture), extraneous_stream_msg.format(\n self._input_depth_mode)\n with tf.name_scope(\"im_stream\"):\n return self._build_im_stream(input_im_node,\n input_pose_node,\n self._im_height,\n self._im_width,\n self._num_channels,\n input_drop_rate_node,\n self._architecture[\"im_stream\"],\n only_stream=True)[0]\n" }, { "path": "gqcnn/search/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\"\"\"\nfrom .search import GQCNNSearch\n\n__all__ = [\"GQCNNSearch\"]\n" }, { "path": "gqcnn/search/enums.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nEnums for hyper-parameter search.\n\nAuthor\n------\nVishal Satish\n\"\"\"\n\n\nclass TrialConstants(object):\n TRIAL_CPU_LOAD = 300 # Decrease to get more aggressize CPU utilization.\n TRIAL_GPU_LOAD = 33 # Decrease to get more aggressize GPU utilization.\n # This really depends on model size (`TRIAL_GPU_LOAD` does too, but it's\n # not a hard limit per se). Ideally we would initialize models one-by-one\n # and monitor the space left, but because model initialization comes after\n # some metric calculation, we set this to be some upper bound based on the\n # largest model and do batch initalizations from there.\n TRIAL_GPU_MEM = 2000\n\n\nclass SearchConstants(object):\n SEARCH_THREAD_SLEEP = 2\n MIN_TIME_BETWEEN_SCHEDULE_ATTEMPTS = 20\n" }, { "path": "gqcnn/search/resource_manager.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nIntelligent resource manager for hyper-parameter search. Queries resources\navailable and appropriately distributes resources over possible trials to run.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nimport math\nimport random\nimport time\n\nimport GPUtil\nimport numpy as np\nimport psutil\n\nfrom autolab_core import Logger\n\nCPU_LOAD_SAMPLE_INTERVAL = 4.0\n# This is a hack because it seems that psutil is returning a lower load than\n# htop, which could be because htop takes into account queued tasks.\nCPU_LOAD_OFFSET = 50\nGPU_STAT_NUM_SAMPLES = 4\nGPU_STAT_SAMPLE_INTERVAL = 1.0\n\n\nclass ResourceManager(object):\n\n def __init__(self,\n trial_cpu_load,\n trial_gpu_load,\n trial_gpu_mem,\n monitor_cpu=True,\n monitor_gpu=True,\n cpu_cores=[],\n gpu_devices=[]):\n self._monitor_cpu = monitor_cpu\n self._monitor_gpu = monitor_gpu\n\n # Set up logger.\n self._logger = Logger.get_logger(self.__class__.__name__)\n\n if not monitor_cpu:\n self._logger.warning(\n \"Not monitoring cpu resources is not advised.\")\n if not monitor_gpu:\n self._logger.warning(\n \"Not monitoring gpu resources is not advised.\")\n\n self._trial_cpu_load = trial_cpu_load\n self._trial_gpu_load = trial_gpu_load\n self._trial_gpu_mem = trial_gpu_mem\n\n self._cpu_cores = cpu_cores\n if len(self._cpu_cores) == 0:\n self._logger.warning(\n \"No CPU cores specified-proceeding to use all available cores.\"\n )\n self._cpu_cores = range(psutil.cpu_count())\n self._cpu_count = len(self._cpu_cores)\n\n self._gpu_devices = gpu_devices\n if len(self._gpu_devices) == 0:\n no_gpus_specified_msg = (\"No GPU devices specified-proceeding to\"\n \" use all available devices.\")\n self._logger.warning(no_gpus_specified_msg)\n self._gpu_devices = range(len(GPUtil.getGPUs()))\n\n @property\n def cpu_cores(self):\n return self._cpu_cores\n\n def _get_cpu_load(self):\n self._logger.info(\"Sampling cpu load...\")\n cpu_core_loads = psutil.cpu_percent(interval=CPU_LOAD_SAMPLE_INTERVAL,\n percpu=True)\n total_load = 0\n for core in self._cpu_cores:\n total_load += cpu_core_loads[core]\n return total_load + CPU_LOAD_OFFSET\n\n def _get_gpu_stats(self):\n self._logger.info(\"Sampling gpu memory and load...\")\n gpu_samples = []\n for _ in range(GPU_STAT_NUM_SAMPLES):\n gpu_samples.append(GPUtil.getGPUs())\n time.sleep(GPU_STAT_SAMPLE_INTERVAL)\n num_gpus = len(gpu_samples[0])\n sample_loads = np.zeros((num_gpus, GPU_STAT_NUM_SAMPLES))\n sample_mems = np.zeros((num_gpus, GPU_STAT_NUM_SAMPLES))\n total_mems = np.zeros((num_gpus, ))\n for i in range(GPU_STAT_NUM_SAMPLES):\n for gpu in gpu_samples[i]:\n if gpu.id in self._gpu_devices:\n sample_loads[gpu.id, i] = gpu.load * 100\n sample_mems[gpu.id, i] = gpu.memoryUsed\n else:\n # Trick the manager into thinking the GPU is fully utilized\n # so it will never be chosen.\n sample_loads[gpu.id, i] = 100\n sample_mems[gpu.id, i] = gpu.memoryTotal\n total_mems[gpu.id] = gpu.memoryTotal\n return total_mems.tolist(), np.mean(\n sample_loads, axis=1).tolist(), np.mean(sample_mems,\n axis=1).tolist()\n\n def _build_gpu_list(self, max_possible_trials_per_device):\n gpus_avail = []\n for device_id, max_trials in enumerate(max_possible_trials_per_device):\n for _ in range(max_trials):\n gpus_avail.append(str(device_id))\n # This is because we might truncate this list later because of a more\n # severe resource bottleneck, in which case we want to evenly\n # distribute the load.\n random.shuffle(gpus_avail)\n return gpus_avail\n\n def num_trials_to_schedule(self, num_pending_trials):\n num_trials_to_schedule = num_pending_trials\n if self._monitor_cpu: # Check cpu bandwith.\n cpu_load = min(self._get_cpu_load(), self._cpu_count * 100)\n max_possible_trials_cpu = int(\n (self._cpu_count * 100 - cpu_load) // self._trial_cpu_load)\n self._logger.info(\"CPU load: {}%, Max possible trials: {}\".format(\n cpu_load, max_possible_trials_cpu))\n num_trials_to_schedule = min(num_trials_to_schedule,\n max_possible_trials_cpu)\n\n if self._monitor_gpu: # Check gpu bandwith.\n total_gpu_mems, gpu_loads, gpu_mems = self._get_gpu_stats()\n max_possible_trials_gpu_load_per_device = [\n int((100 - gpu_load) // self._trial_gpu_load)\n for gpu_load in gpu_loads\n ]\n max_possible_trials_gpu_mem_per_device = [\n int((total_gpu_mem - gpu_mem) // self._trial_gpu_mem)\n for total_gpu_mem, gpu_mem in zip(total_gpu_mems, gpu_mems)\n ]\n max_possible_trials_gpu_per_device = map(\n lambda x: min(x[0], x[1]),\n zip(max_possible_trials_gpu_load_per_device,\n max_possible_trials_gpu_mem_per_device))\n self._logger.info(\n \"GPU loads: {}, GPU mems: {}, Max possible trials: {}\".format(\n \"% \".join([str(gpu_load) for gpu_load in gpu_loads]) + \"%\",\n \"MiB \".join([str(gpu_mem)\n for gpu_mem in gpu_mems]) + \"MiB\",\n sum(max_possible_trials_gpu_per_device)))\n num_trials_to_schedule = min(\n num_trials_to_schedule,\n sum(max_possible_trials_gpu_per_device))\n\n # Build the device list for scheduling trials on specific gpus.\n gpus_avail = self._build_gpu_list(\n max_possible_trials_gpu_per_device)\n else:\n # Just distribute load among gpus.\n num_gpus = self._get_gpu_count()\n trials_per_gpu = int(math.ceil(num_trials_to_schedule / num_gpus))\n gpus_avail = self._build_gpu_list([trials_per_gpu] * num_gpus)\n\n gpus_avail = gpus_avail[:num_trials_to_schedule]\n self._logger.info(\n \"Max possible trials overall: {}\".format(num_trials_to_schedule))\n return num_trials_to_schedule, gpus_avail\n" }, { "path": "gqcnn/search/search.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nPerform hyper-parameter search over a set of GQ-CNN model/training\nparameters. Actively monitor system resources and appropriately schedule\ntrials.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nimport os\nimport time\n\nfrom .resource_manager import ResourceManager\nfrom .trial import (GQCNNTrainingAndAnalysisTrial,\n GQCNNFineTuningAndAnalysisTrial)\nfrom .utils import gen_trial_params, gen_timestamp, log_trial_status\nfrom .enums import TrialConstants, SearchConstants\n\nfrom autolab_core import Logger\n\nfrom ..utils import is_py2, GQCNNTrainingStatus\n\nif is_py2():\n from Queue import Queue\nelse:\n from queue import Queue\n\n\nclass GQCNNSearch(object):\n\n def __init__(self,\n analysis_config,\n train_configs,\n datasets,\n split_names,\n base_models=[],\n output_dir=None,\n search_name=None,\n monitor_cpu=True,\n monitor_gpu=True,\n cpu_cores=[],\n gpu_devices=[]):\n self._analysis_cfg = analysis_config\n\n # Create trial output dir if not specified.\n if search_name is None:\n search_name = \"gqcnn_hyperparam_search_{}\".format(gen_timestamp())\n if output_dir is None:\n output_dir = \"models\"\n self._trial_output_dir = os.path.join(output_dir, search_name)\n if not os.path.exists(self._trial_output_dir):\n os.makedirs(self._trial_output_dir)\n\n # Set up logger.\n self._logger = Logger.get_logger(self.__class__.__name__,\n log_file=os.path.join(\n self._trial_output_dir,\n \"search.log\"),\n global_log_file=True)\n\n # Init resource manager.\n self._resource_manager = ResourceManager(TrialConstants.TRIAL_CPU_LOAD,\n TrialConstants.TRIAL_GPU_LOAD,\n TrialConstants.TRIAL_GPU_MEM,\n monitor_cpu=monitor_cpu,\n monitor_gpu=monitor_gpu,\n cpu_cores=cpu_cores,\n gpu_devices=gpu_devices)\n\n # Parse train configs and generate individual trial parameters.\n if len(base_models) > 0:\n inconsistent_inputs_msg = (\"Must have equal number of training\"\n \" configs, datasets, split_names, and\"\n \" base models!\")\n assert len(train_configs) == len(datasets) == len(\n split_names) == len(base_models), inconsistent_inputs_msg\n else:\n inconsistent_inputs_msg = (\"Must have equal number of training\"\n \" configs, datasets, and split_names!\")\n assert len(train_configs) == len(datasets) == len(\n split_names), inconsistent_inputs_msg\n self._logger.info(\"Generating trial parameters...\")\n trial_params = gen_trial_params(train_configs,\n datasets,\n split_names,\n base_models=base_models)\n\n # Create pending trial queue.\n self._trials_pending_queue = Queue()\n if len(base_models) > 0:\n for trial_name, hyperparam_summary, train_cfg, dataset, \\\n base_model, split_name in trial_params:\n self._trials_pending_queue.put(\n GQCNNFineTuningAndAnalysisTrial(self._analysis_cfg,\n train_cfg, dataset,\n base_model, split_name,\n self._trial_output_dir,\n trial_name,\n hyperparam_summary))\n else:\n for trial_name, hyperparam_summary, train_cfg, dataset, \\\n split_name in trial_params:\n self._trials_pending_queue.put(\n GQCNNTrainingAndAnalysisTrial(self._analysis_cfg,\n train_cfg, dataset,\n split_name,\n self._trial_output_dir,\n trial_name,\n hyperparam_summary))\n\n # Create containers to hold running, finished, and errored-out trials.\n self._trials_running = []\n self._trials_finished = []\n self._trials_errored = []\n\n def search(self):\n self._logger.info(\"Beginning hyper-parameter search...\")\n done = False\n waiting_for_trial_init = False\n last_schedule_attempt_time = -1\n search_start_time = time.time()\n while not done:\n num_trials_pending = self._trials_pending_queue.qsize()\n num_trials_running = len(self._trials_running)\n num_trials_finished = len(self._trials_finished)\n num_trials_errored = len(self._trials_errored)\n\n self._logger.info(\n \"----------------------------------------------------\")\n self._logger.info(\n \"Num trials pending: {}\".format(num_trials_pending))\n self._logger.info(\n \"Num trials running: {}\".format(num_trials_running))\n self._logger.info(\n \"Num trials finished: {}\".format(num_trials_finished))\n if num_trials_errored > 0:\n self._logger.info(\n \"Num trials errored: {}\".format(num_trials_errored))\n\n if num_trials_pending > 0 and not waiting_for_trial_init and (\n time.time() - last_schedule_attempt_time\n ) > SearchConstants.MIN_TIME_BETWEEN_SCHEDULE_ATTEMPTS:\n self._logger.info(\"Attempting to schedule more trials...\")\n num_trials_to_schedule, gpus_avail = \\\n self._resource_manager.num_trials_to_schedule(\n num_trials_pending)\n self._logger.info(\n \"Scheduling {} trials\".format(num_trials_to_schedule))\n\n if num_trials_to_schedule > 0:\n # Start trials.\n for _, gpu in zip(range(num_trials_to_schedule),\n gpus_avail):\n trial = self._trials_pending_queue.get()\n trial.begin(\n gpu_avail=gpu,\n cpu_cores_avail=self._resource_manager.cpu_cores)\n self._trials_running.append(trial)\n\n # Block scheduling until trials have started training (this\n # is when we know what resources are still available).\n waiting_for_trial_init = True\n last_schedule_attempt_time = time.time()\n\n # Check if trials have started training.\n if waiting_for_trial_init:\n training_has_started = [\n trial.training_status == GQCNNTrainingStatus.TRAINING\n for trial in self._trials_running\n ]\n if all(training_has_started):\n waiting_for_trial_init = False\n\n # Log trial status.\n if len(self._trials_running) > 0:\n self._logger.info(log_trial_status(self._trials_running))\n\n # Check if any trials have finished running or errored-out.\n finished_trials_to_move = []\n errored_trials_to_move = []\n for trial in self._trials_running:\n if trial.finished:\n finished_trials_to_move.append(trial)\n elif trial.errored_out:\n errored_trials_to_move.append(trial)\n self._trials_finished.extend(finished_trials_to_move)\n self._trials_errored.extend(errored_trials_to_move)\n for trial in finished_trials_to_move:\n self._trials_running.remove(trial)\n for trial in errored_trials_to_move:\n self._trials_running.remove(trial)\n\n # Update stopping criteria and sleep.\n done = (num_trials_pending == 0) and (num_trials_running == 0)\n time.sleep(SearchConstants.SEARCH_THREAD_SLEEP)\n\n self._logger.info(\n \"------------------Successful Trials------------------\")\n self._logger.info(log_trial_status(self._trials_finished))\n if len(self._trials_errored) > 0:\n self._logger.info(\n \"--------------------Failed Trials--------------------\")\n self._logger.info(log_trial_status(self._trials_errored))\n\n self._logger.info(\n \"Hyper-parameter search finished in {} seconds.\".format(\n time.time() - search_start_time))\n" }, { "path": "gqcnn/search/trial.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nTrials for hyper-parameter search.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nfrom abc import ABC, abstractmethod\nimport json\nimport multiprocessing\nimport os\nimport sys\n\nimport numpy as np\n\nfrom ..model import get_gqcnn_model\nfrom ..training import get_gqcnn_trainer\nfrom ..utils import GeneralConstants, GQCNNTrainingStatus\nfrom ..analysis import GQCNNAnalyzer\n\n\nclass TrialStatus:\n PENDING = \"pending\"\n RUNNING = \"running\"\n FINISHED = \"finished\"\n EXCEPTION = \"exception\"\n\n\nclass GQCNNTrialWithAnalysis(ABC):\n\n def __init__(self, analysis_cfg, train_cfg, dataset_dir, split_name,\n output_dir, model_name, hyperparam_summary):\n self._analysis_cfg = analysis_cfg\n self._train_cfg = train_cfg\n self._dataset_dir = dataset_dir\n self._split_name = split_name\n self._output_dir = output_dir\n self._model_name = model_name\n self._hyperparam_summary = hyperparam_summary\n self._manager = multiprocessing.Manager()\n # To communicate with training.\n self._train_progress_dict = self._build_train_progress_dict()\n # To communicate with trial.\n self._trial_progress_dict = self._build_trial_progress_dict()\n self._process = None\n\n def _build_train_progress_dict(self):\n progress_dict = self._manager.dict(\n training_status=GQCNNTrainingStatus.NOT_STARTED,\n epoch=np.nan,\n analysis=None)\n return progress_dict\n\n def _build_trial_progress_dict(self):\n progress_dict = self._manager.dict(status=TrialStatus.PENDING)\n return progress_dict\n\n @abstractmethod\n def _run(self, trainer):\n pass\n\n def _run_trial(self,\n analysis_config,\n train_config,\n dataset_dir,\n split_name,\n output_dir,\n model_name,\n train_progress_dict,\n trial_progress_dict,\n hyperparam_summary,\n gpu_avail=\"\",\n cpu_cores_avail=[],\n backend=\"tf\"):\n trial_progress_dict[\"status\"] = TrialStatus.RUNNING\n try:\n os.system(\"taskset -pc {} {}\".format(\n \",\".join(str(i) for i in cpu_cores_avail), os.getpid()))\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = gpu_avail\n\n gqcnn = get_gqcnn_model(backend,\n verbose=False)(train_config[\"gqcnn\"],\n verbose=False)\n trainer = get_gqcnn_trainer(backend)(\n gqcnn,\n dataset_dir,\n split_name,\n output_dir,\n train_config,\n name=model_name,\n progress_dict=train_progress_dict,\n verbose=False)\n self._run(trainer)\n\n with open(\n os.path.join(output_dir, model_name,\n \"hyperparam_summary.json\"), \"wb\") as fhandle:\n json.dump(hyperparam_summary,\n fhandle,\n indent=GeneralConstants.JSON_INDENT)\n\n train_progress_dict[\"training_status\"] = \"analyzing\"\n analyzer = GQCNNAnalyzer(analysis_config, verbose=False)\n _, _, init_train_error, final_train_error, init_train_loss, \\\n final_train_loss, init_val_error, final_val_error, \\\n norm_final_val_error = analyzer.analyze(\n os.path.join(output_dir, model_name), output_dir)\n analysis_dict = {}\n analysis_dict[\"init_train_error\"] = init_train_error\n analysis_dict[\"final_train_error\"] = final_train_error\n analysis_dict[\"init_train_loss\"] = init_train_loss\n analysis_dict[\"final_train_loss\"] = final_train_loss\n analysis_dict[\"init_val_error\"] = init_val_error\n analysis_dict[\"final_val_error\"] = final_val_error\n analysis_dict[\"norm_final_val_error\"] = norm_final_val_error\n train_progress_dict[\"analysis\"] = analysis_dict\n\n train_progress_dict[\"training_status\"] = \"finished\"\n trial_progress_dict[\"status\"] = TrialStatus.FINISHED\n sys.exit(0)\n except Exception as e:\n trial_progress_dict[\"status\"] = TrialStatus.EXCEPTION\n trial_progress_dict[\"error_msg\"] = str(e)\n sys.exit(0)\n\n @property\n def finished(self):\n return self._trial_progress_dict[\"status\"] == TrialStatus.FINISHED\n\n @property\n def errored_out(self):\n return self._trial_progress_dict[\"status\"] == TrialStatus.EXCEPTION\n\n @property\n def error_msg(self):\n return self._trial_progress_dict[\"error_msg\"]\n\n @property\n def training_status(self):\n return self._train_progress_dict[\"training_status\"]\n\n def begin(self, gpu_avail=\"\", cpu_cores_avail=[]):\n self._status = TrialStatus.RUNNING\n self._process = multiprocessing.Process(\n target=self._run_trial,\n args=(self._analysis_cfg, self._train_cfg, self._dataset_dir,\n self._split_name, self._output_dir, self._model_name,\n self._train_progress_dict, self._trial_progress_dict,\n self._hyperparam_summary, gpu_avail, cpu_cores_avail))\n self._process.start()\n\n def __str__(self):\n trial_str = \"Trial: {}, Training Stage: {}\".format(\n self._model_name, self.training_status)\n if self.training_status == GQCNNTrainingStatus.TRAINING and not \\\n np.isnan(self._train_progress_dict[\"epoch\"]):\n trial_str += \", Epoch: {}/{}\".format(\n self._train_progress_dict[\"epoch\"],\n self._train_cfg[\"num_epochs\"])\n if self.errored_out:\n trial_str += \", Error message: {}\".format(self.error_msg)\n if self.training_status == \"finished\":\n finished_msg = (\", Initial train error: {}, Final train error: {},\"\n \" Initial train loss: {}, Final train loss: {},\"\n \" Initial val error: {}, Final val error: {}, Norm\"\n \" final val error: {}\")\n trial_str += finished_msg.format(\n self._train_progress_dict[\"analysis\"][\"init_train_error\"],\n self._train_progress_dict[\"analysis\"][\"final_train_error\"],\n self._train_progress_dict[\"analysis\"][\"init_train_loss\"],\n self._train_progress_dict[\"analysis\"][\"final_train_loss\"],\n self._train_progress_dict[\"analysis\"][\"init_val_error\"],\n self._train_progress_dict[\"analysis\"][\"final_val_error\"],\n self._train_progress_dict[\"analysis\"][\"norm_final_val_error\"])\n return trial_str\n\n\nclass GQCNNTrainingAndAnalysisTrial(GQCNNTrialWithAnalysis):\n\n def _run(self, trainer):\n trainer.train()\n\n\nclass GQCNNFineTuningAndAnalysisTrial(GQCNNTrialWithAnalysis):\n\n def __init__(self, analysis_cfg, train_cfg, dataset_dir, base_model_dir,\n split_name, output_dir, model_name, hyperparam_summary):\n GQCNNTrialWithAnalysis.__init__(self, analysis_cfg, train_cfg,\n dataset_dir, split_name, output_dir,\n model_name, hyperparam_summary)\n\n self._base_model_dir = base_model_dir\n\n def _run(self, trainer):\n trainer.finetune(self._base_model_dir)\n" }, { "path": "gqcnn/search/utils.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nUtility functions for hyper-parameter search.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nfrom collections import OrderedDict, defaultdict\nimport copy\nfrom datetime import datetime\nimport itertools\n\n\ndef get_fields_to_search_over(train_config, prev_keys=[]):\n fields = []\n anchored_fields = defaultdict(list)\n for key in train_config:\n if isinstance(train_config[key], list):\n prev_keys_copy = copy.deepcopy(prev_keys)\n prev_keys_copy.append(key)\n if isinstance(train_config[key][0],\n str) and train_config[key][0].startswith(\"anchor_\"):\n anchored_fields[train_config[key][0]].append(prev_keys_copy)\n train_config[key] = train_config[key][1:]\n else:\n fields.append(prev_keys_copy)\n elif isinstance(train_config[key], OrderedDict):\n prev_keys_copy = copy.deepcopy(prev_keys)\n prev_keys_copy.append(key)\n sub_fields, sub_anchored_fields = get_fields_to_search_over(\n train_config[key], prev_keys=prev_keys_copy)\n fields.extend(sub_fields)\n update_dict(anchored_fields, sub_anchored_fields)\n return fields, anchored_fields\n\n\ndef update_dict(dict1, dict2):\n for key, val in dict2.items():\n if key in dict1:\n dict1[key].extend(val)\n else:\n dict1[key] = val\n\n\ndef get_nested_key(cfg, key):\n val = cfg\n for k in key:\n val = val[k]\n return val\n\n\ndef set_nested_key(cfg, key, val):\n root_field = cfg\n for k in key[:-1]:\n root_field = root_field[k]\n root_field[key[-1]] = val\n\n\ndef gen_config_summary_dict(hyperparam_combination):\n summary_dict = {}\n for key, val in hyperparam_combination:\n summary_dict[\"/\".join(key)] = val\n return summary_dict\n\n\ndef parse_master_train_config(train_config):\n configs = []\n hyperparam_search_fields, hyperparam_anchored_search_fields = \\\n get_fields_to_search_over(train_config)\n\n # Ensure a one-to-one mapping between hyperparameters of fields with\n # matching anchor tags.\n for anchor_tag, fields in hyperparam_anchored_search_fields.items():\n num_params = []\n for field in fields:\n num_params.append(len(get_nested_key(train_config, field)))\n invalid_anchor_tag_msg = (\"All fields in anchor tag '{}' do not have\"\n \" the same # of parameters to search over!\")\n assert max(num_params) == min(\n num_params), invalid_anchor_tag_msg.format(anchor_tag)\n\n # If there is nothing to search over just return the given config.\n if len(hyperparam_search_fields) == 0 and len(\n hyperparam_anchored_search_fields) == 0:\n return [(\"\", train_config)]\n\n # Generate a list of all the possible hyper-parameters to search over.\n # Normal fields.\n hyperparam_search_params = []\n for search_field in hyperparam_search_fields:\n search_field_params = []\n for val in get_nested_key(train_config, search_field):\n search_field_params.append((search_field, val))\n hyperparam_search_params.append(search_field_params)\n # Anchored fields.\n for anchored_fields in hyperparam_anchored_search_fields.values():\n combinations = [[] for _ in range(\n len(get_nested_key(train_config, anchored_fields[0])))]\n for field in anchored_fields:\n for idx, val in enumerate(get_nested_key(train_config, field)):\n combinations[idx].append((field, val))\n hyperparam_search_params.append(combinations)\n\n # Get all permutations of the possible hyper-parameters.\n hyperparam_combinations = list(\n itertools.product(*hyperparam_search_params))\n\n def flatten_combo(combo):\n flattened = []\n for item in combo:\n if isinstance(item, list):\n for sub_item in item:\n flattened.append(sub_item)\n else:\n flattened.append(item)\n return flattened\n\n hyperparam_combinations = [\n flatten_combo(combo) for combo in hyperparam_combinations\n ]\n\n # Generate possible configs to search over.\n for combo in hyperparam_combinations:\n config = copy.deepcopy(train_config)\n for field, val in combo:\n set_nested_key(config, field, val)\n configs.append((gen_config_summary_dict(combo), config))\n return configs\n\n\ndef gen_timestamp():\n return str(datetime.now()).split(\".\")[0].replace(\" \", \"_\")\n\n\ndef gen_trial_params_train(master_train_configs, datasets, split_names):\n trial_params = []\n for master_train_config, dataset, split_name in zip(\n master_train_configs, datasets, split_names):\n train_configs = parse_master_train_config(master_train_config)\n for i, (hyperparam_summary_dict,\n train_config) in enumerate(train_configs):\n trial_name = \"{}_{}_trial_{}_{}\".format(\n dataset.split(\"/\")[-3], split_name, i, gen_timestamp())\n trial_params.append((trial_name, hyperparam_summary_dict,\n train_config, dataset, split_name))\n return trial_params\n\n\ndef gen_trial_params_finetune(master_train_configs, datasets, base_models,\n split_names):\n trial_params = []\n for master_train_config, dataset, base_model, split_name in zip(\n master_train_configs, datasets, base_models, split_names):\n train_configs = parse_master_train_config(master_train_config)\n for i, (hyperparam_summary_dict,\n train_config) in enumerate(train_configs):\n trial_name = \"{}_{}_trial_{}_{}\".format(\n dataset.split(\"/\")[-3], split_name, i, gen_timestamp())\n trial_params.append(\n (trial_name, hyperparam_summary_dict, train_config, dataset,\n base_model, split_name))\n return trial_params\n\n\ndef gen_trial_params(master_train_configs,\n datasets,\n split_names,\n base_models=[]):\n if len(base_models) > 0:\n return gen_trial_params_finetune(master_train_configs, datasets,\n base_models, split_names)\n else:\n return gen_trial_params_train(master_train_configs, datasets,\n split_names)\n\n\ndef log_trial_status(trials):\n status_str = \"--------------------TRIAL STATUS--------------------\"\n for trial in trials:\n status_str += \"\\n\"\n status_str += \"[{}]\".format(str(trial))\n return status_str\n" }, { "path": "gqcnn/training/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nFactory functions to obtain GQCNNTrainer class based on chosen deep learning\nbackend. Currently only Tensorflow is supported.\nAuthor: Vishal Satish\n\"\"\"\nfrom .tf import GQCNNTrainerTF\n\n\ndef get_gqcnn_trainer(backend=\"tf\"):\n \"\"\"Get the GQ-CNN Trainer for the provided backend.\n\n Note\n ----\n Currently only TensorFlow is supported.\n\n Parameters\n ----------\n backend : str\n The backend to use, currently only \"tf\" is supported.\n\n Returns\n -------\n :obj:`gqcnn.training.tf.GQCNNTrainerTF`\n GQ-CNN Trainer with TensorFlow backend.\n \"\"\"\n # Return desired `GQCNNTrainer` instance based on backend.\n if backend == \"tf\":\n return GQCNNTrainerTF\n else:\n raise ValueError(\"Invalid backend: {}\".format(backend))\n" }, { "path": "gqcnn/training/tf/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\"\"\"\nfrom .trainer_tf import GQCNNTrainerTF\n\n__all__ = [\"GQCNNTrainerTF\"]\n" }, { "path": "gqcnn/training/tf/trainer_tf.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nTrains a GQ-CNN network using Tensorflow backend.\n\nAuthor\n------\nVishal Satish & Jeff Mahler\n\"\"\"\nimport collections\nimport json\nimport multiprocessing as mp\nimport os\nimport random\nimport shutil\nimport signal\nimport subprocess\nimport sys\nimport time\n\nfrom past.builtins import xrange\nimport cv2\nimport numpy as np\nimport scipy.stats as ss\nimport tensorflow as tf\n\nfrom autolab_core import (BinaryClassificationResult, RegressionResult,\n TensorDataset, Logger)\nfrom autolab_core.constants import JSON_INDENT\nimport autolab_core.utils as utils\n\nfrom ...utils import (TrainingMode, GripperMode, InputDepthMode,\n GeneralConstants, TrainStatsLogger, GQCNNTrainingStatus,\n GQCNNFilenames, pose_dim, read_pose_data,\n weight_name_to_layer_name, is_py2, imresize)\n\nif is_py2():\n import Queue\nelse:\n import queue as Queue\n\n\nclass GQCNNTrainerTF(object):\n \"\"\"Trains a GQ-CNN with Tensorflow backend.\"\"\"\n\n def __init__(self,\n gqcnn,\n dataset_dir,\n split_name,\n output_dir,\n config,\n name=None,\n progress_dict=None,\n verbose=True):\n \"\"\"\n Parameters\n ----------\n gqcnn : :obj:`GQCNN`\n Grasp quality neural network to optimize.\n dataset_dir : str\n Path to the training/validation dataset.\n split_name : str\n Name of the split to train on.\n output_dir : str\n Path to save the model output.\n config : dict\n Dictionary of configuration parameters.\n name : str\n Name of the the model.\n \"\"\"\n self.gqcnn = gqcnn\n self.dataset_dir = dataset_dir\n self.split_name = split_name\n self.output_dir = output_dir\n self.cfg = config\n self.tensorboard_has_launched = False\n self.model_name = name\n self.progress_dict = progress_dict\n self.finetuning = False\n\n # Create a directory for the model.\n if self.model_name is None:\n model_id = utils.gen_experiment_id()\n self.model_name = \"model_{}\".format(model_id)\n self.model_dir = os.path.join(self.output_dir, self.model_name)\n if not os.path.exists(self.model_dir):\n os.mkdir(self.model_dir)\n\n # Set up logger.\n self.logger = Logger.get_logger(self.__class__.__name__,\n log_file=os.path.join(\n self.model_dir, \"training.log\"),\n silence=(not verbose),\n global_log_file=verbose)\n\n # Check default split.\n if split_name is None:\n self.logger.warning(\"Using default image-wise split.\")\n self.split_name = \"image_wise\"\n\n # Update cfg for saving.\n self.cfg[\"dataset_dir\"] = self.dataset_dir\n self.cfg[\"split_name\"] = self.split_name\n\n def _create_loss(self):\n \"\"\"Creates a loss based on config file.\n\n Returns\n -------\n :obj:`tensorflow Tensor`\n Loss.\n \"\"\"\n if self.cfg[\"loss\"] == \"l2\":\n return (1.0 / self.train_batch_size) * tf.nn.l2_loss(\n tf.subtract(tf.nn.sigmoid(self.train_net_output),\n self.train_labels_node))\n elif self.cfg[\"loss\"] == \"sparse\":\n if self._angular_bins > 0:\n log = tf.reshape(\n tf.dynamic_partition(self.train_net_output,\n self.train_pred_mask_node, 2)[1],\n (-1, 2))\n return tf.reduce_mean(\n tf.nn.sparse_softmax_cross_entropy_with_logits(\n _sentinel=None,\n labels=self.train_labels_node,\n logits=log))\n else:\n return tf.reduce_mean(\n tf.nn.sparse_softmax_cross_entropy_with_logits(\n _sentinel=None,\n labels=self.train_labels_node,\n logits=self.train_net_output,\n name=None))\n elif self.cfg[\"loss\"] == \"weighted_cross_entropy\":\n return tf.reduce_mean(\n tf.nn.weighted_cross_entropy_with_logits(\n targets=tf.reshape(self.train_labels_node, [-1, 1]),\n logits=self.train_net_output,\n pos_weight=self.pos_weight,\n name=None))\n\n def _create_optimizer(self, loss, batch, var_list, learning_rate):\n \"\"\"Create optimizer based on config file.\n\n Parameters\n ----------\n loss : :obj:`tensorflow Tensor`\n Loss to use, generated with `_create_loss`.\n batch : :obj:`tf.Variable`\n Variable to keep track of the current gradient step number.\n var_list : :obj:`lst`\n List of tf.Variable objects to update to minimize loss (ex. network\n weights).\n learning_rate : float\n Learning rate for training.\n\n Returns\n -------\n :obj:`tf.train.Optimizer`\n Optimizer.\n \"\"\"\n # Instantiate optimizer.\n if self.cfg[\"optimizer\"] == \"momentum\":\n optimizer = tf.train.MomentumOptimizer(learning_rate,\n self.momentum_rate)\n elif self.cfg[\"optimizer\"] == \"adam\":\n optimizer = tf.train.AdamOptimizer(learning_rate)\n elif self.cfg[\"optimizer\"] == \"rmsprop\":\n optimizer = tf.train.RMSPropOptimizer(learning_rate)\n else:\n raise ValueError(\"Optimizer '{}' not supported\".format(\n self.cfg[\"optimizer\"]))\n\n # Compute gradients.\n gradients, variables = zip(\n *optimizer.compute_gradients(loss, var_list=var_list))\n\n # Clip gradients to prevent exploding gradient problem.\n gradients, global_grad_norm = tf.clip_by_global_norm(\n gradients, self.max_global_grad_norm)\n\n # Generate op to apply gradients.\n apply_grads = optimizer.apply_gradients(zip(gradients, variables),\n global_step=batch)\n\n return apply_grads, global_grad_norm\n\n def _launch_tensorboard(self):\n \"\"\"Launches Tensorboard to visualize training.\"\"\"\n FNULL = open(os.devnull, \"w\")\n tensorboard_launch_msg = (\"Launching Tensorboard, please navigate to\"\n \" localhost:{} in your favorite web browser\"\n \" to view summaries.\")\n self.logger.info(tensorboard_launch_msg.format(self._tensorboard_port))\n self._tensorboard_proc = subprocess.Popen([\n \"tensorboard\", \"--port\",\n str(self._tensorboard_port), \"--logdir\", self.summary_dir\n ],\n stdout=FNULL)\n\n def _close_tensorboard(self):\n \"\"\"Closes Tensorboard process.\"\"\"\n self.logger.info(\"Closing Tensorboard...\")\n self._tensorboard_proc.terminate()\n\n def train(self):\n \"\"\"Perform optimization.\"\"\"\n with self.gqcnn.tf_graph.as_default():\n self._train()\n\n def _train(self):\n \"\"\"Perform optimization.\"\"\"\n # Run setup.\n if self.progress_dict is not None:\n self.progress_dict[\n \"training_status\"] = GQCNNTrainingStatus.SETTING_UP\n self._setup()\n\n # Build network.\n self.gqcnn.initialize_network(self.input_im_node, self.input_pose_node)\n\n # Optimize weights.\n if self.progress_dict is not None:\n self.progress_dict[\n \"training_status\"] = GQCNNTrainingStatus.TRAINING\n self._optimize_weights()\n\n def finetune(self, base_model_dir):\n \"\"\"Perform fine-tuning.\n\n Parameters\n ----------\n base_model_dir : str\n Path to the pre-trained base model to use.\n \"\"\"\n with self.gqcnn.tf_graph.as_default():\n self._finetune(base_model_dir)\n\n def _finetune(self, base_model_dir):\n \"\"\"Perform fine-tuning.\n\n Parameters\n ----------\n base_model_dir : str\n Path to the pre-trained base model to use.\n \"\"\"\n # Set flag and base model for fine-tuning.\n self.finetuning = True\n self.base_model_dir = base_model_dir\n\n # Run setup.\n self._setup()\n\n # Build network.\n self.gqcnn.set_base_network(base_model_dir)\n self.gqcnn.initialize_network(self.input_im_node, self.input_pose_node)\n\n # Optimize weights.\n if self.progress_dict is not None:\n self.progress_dict[\n \"training_status\"] = GQCNNTrainingStatus.TRAINING\n self._optimize_weights(finetune=True)\n\n def _optimize_weights(self, finetune=False):\n \"\"\"Optimize the network weights.\"\"\"\n start_time = time.time()\n\n # Setup output.\n self.train_net_output = self.gqcnn.output\n if self.training_mode == TrainingMode.CLASSIFICATION:\n if self.cfg[\"loss\"] == \"weighted_cross_entropy\":\n self.gqcnn.add_sigmoid_to_output()\n else:\n self.gqcnn.add_softmax_to_output()\n elif self.training_mode == TrainingMode.REGRESSION:\n self.gqcnn.add_sigmoid_to_output()\n else:\n raise ValueError(\"Training mode: {} not supported !\".format(\n self.training_mode))\n train_predictions = self.gqcnn.output\n drop_rate_in = self.gqcnn.input_drop_rate_node\n self.weights = self.gqcnn.weights\n\n # Once weights have been initialized, create TF saver for weights.\n self.saver = tf.train.Saver()\n\n # Form loss.\n with tf.name_scope(\"loss\"):\n # Part 1: error.\n loss = self._create_loss()\n unregularized_loss = loss\n\n # Part 2: regularization.\n layer_weights = list(self.weights.values())\n with tf.name_scope(\"regularization\"):\n regularizers = tf.nn.l2_loss(layer_weights[0])\n for w in layer_weights[1:]:\n regularizers = regularizers + tf.nn.l2_loss(w)\n loss += self.train_l2_regularizer * regularizers\n\n # Setup learning rate.\n batch = tf.Variable(0)\n learning_rate = tf.train.exponential_decay(\n self.base_lr, # Base learning rate.\n batch * self.train_batch_size, # Current index into the dataset.\n self.decay_step, # Decay step.\n self.decay_rate, # decay rate.\n staircase=True)\n\n # Setup variable list.\n var_list = list(self.weights.values())\n if finetune:\n var_list = []\n for weights_name, weights_val in self.weights.items():\n layer_name = weight_name_to_layer_name(weights_name)\n if self.optimize_base_layers or \\\n layer_name not in self.gqcnn._base_layer_names:\n var_list.append(weights_val)\n\n # Create optimizer.\n with tf.name_scope(\"optimizer\"):\n apply_grad_op, global_grad_norm = self._create_optimizer(\n loss, batch, var_list, learning_rate)\n\n # Add a handler for SIGINT for graceful exit.\n def handler(signum, frame):\n self.logger.info(\"caught CTRL+C, exiting...\")\n self._cleanup()\n exit(0)\n\n signal.signal(signal.SIGINT, handler)\n\n # Now that everything in our graph is set up, we write the graph to the\n # summary event so it can be visualized in Tensorboard.\n self.summary_writer.add_graph(self.gqcnn.tf_graph)\n\n # Begin optimization loop.\n try:\n # Start prefetch queue workers.\n self.prefetch_q_workers = []\n seed = self._seed\n for i in range(self.num_prefetch_q_workers):\n if self.num_prefetch_q_workers > 1 or not self._debug:\n seed = np.random.randint(GeneralConstants.SEED_SAMPLE_MAX)\n p = mp.Process(target=self._load_and_enqueue, args=(seed, ))\n p.start()\n self.prefetch_q_workers.append(p)\n\n # Init TF variables.\n init = tf.global_variables_initializer()\n self.sess.run(init)\n\n self.logger.info(\"Beginning Optimization...\")\n\n # Create a `TrainStatsLogger` object to log training statistics at\n # certain intervals.\n self.train_stats_logger = TrainStatsLogger(self.model_dir)\n\n # Loop through training steps.\n training_range = xrange(\n int(self.num_epochs * self.num_train) // self.train_batch_size)\n for step in training_range:\n # Run optimization.\n step_start = time.time()\n if self._angular_bins > 0:\n images, poses, labels, masks = self.prefetch_q.get()\n _, l, ur_l, lr, predictions, raw_net_output = \\\n self.sess.run(\n [\n apply_grad_op, loss, unregularized_loss,\n learning_rate, train_predictions,\n self.train_net_output\n ],\n feed_dict={\n drop_rate_in: self.drop_rate,\n self.input_im_node: images,\n self.input_pose_node: poses,\n self.train_labels_node: labels,\n self.train_pred_mask_node: masks\n },\n options=GeneralConstants.timeout_option)\n else:\n images, poses, labels = self.prefetch_q.get()\n _, l, ur_l, lr, predictions, raw_net_output = \\\n self.sess.run(\n [\n apply_grad_op, loss, unregularized_loss,\n learning_rate, train_predictions,\n self.train_net_output\n ],\n feed_dict={\n drop_rate_in: self.drop_rate,\n self.input_im_node: images,\n self.input_pose_node: poses,\n self.train_labels_node: labels\n },\n options=GeneralConstants.timeout_option)\n step_stop = time.time()\n self.logger.info(\"Step took {} sec.\".format(\n str(round(step_stop - step_start, 3))))\n\n if self.training_mode == TrainingMode.REGRESSION:\n self.logger.info(\"Max \" + str(np.max(predictions)))\n self.logger.info(\"Min \" + str(np.min(predictions)))\n elif self.cfg[\"loss\"] != \"weighted_cross_entropy\":\n if self._angular_bins == 0:\n ex = np.exp(raw_net_output - np.tile(\n np.max(raw_net_output, axis=1)[:, np.newaxis],\n [1, 2]))\n softmax = ex / np.tile(\n np.sum(ex, axis=1)[:, np.newaxis], [1, 2])\n\n self.logger.info(\"Max \" + str(np.max(softmax[:, 1])))\n self.logger.info(\"Min \" + str(np.min(softmax[:, 1])))\n self.logger.info(\"Pred nonzero \" +\n str(np.sum(softmax[:, 1] > 0.5)))\n self.logger.info(\"True nonzero \" + str(np.sum(labels)))\n\n else:\n sigmoid = 1.0 / (1.0 + np.exp(-raw_net_output))\n self.logger.info(\"Max \" + str(np.max(sigmoid)))\n self.logger.info(\"Min \" + str(np.min(sigmoid)))\n self.logger.info(\"Pred nonzero \" +\n str(np.sum(sigmoid > 0.5)))\n self.logger.info(\"True nonzero \" +\n str(np.sum(labels > 0.5)))\n\n if np.isnan(l) or np.any(np.isnan(poses)):\n self.logger.error(\n \"Encountered NaN in loss or training poses!\")\n raise Exception\n\n # Log output.\n if step % self.log_frequency == 0:\n elapsed_time = time.time() - start_time\n start_time = time.time()\n self.logger.info(\"Step {} (epoch {}), {} s\".format(\n step,\n str(\n round(\n step * self.train_batch_size / self.num_train,\n 3)),\n str(round(1000 * elapsed_time / self.eval_frequency,\n 2))))\n self.logger.info(\n \"Minibatch loss: {}, learning rate: {}\".format(\n str(round(l, 3)), str(round(lr, 6))))\n if self.progress_dict is not None:\n self.progress_dict[\"epoch\"] = str(\n round(\n step * self.train_batch_size / self.num_train,\n 2))\n\n train_error = l\n if self.training_mode == TrainingMode.CLASSIFICATION:\n if self._angular_bins > 0:\n predictions = predictions[masks.astype(\n bool)].reshape((-1, 2))\n classification_result = BinaryClassificationResult(\n predictions[:, 1], labels)\n train_error = classification_result.error_rate\n\n self.logger.info(\"Minibatch error: {}\".format(\n str(round(train_error, 3))))\n\n self.summary_writer.add_summary(\n self.sess.run(\n self.merged_log_summaries,\n feed_dict={\n self.minibatch_error_placeholder: train_error,\n self.minibatch_loss_placeholder: l,\n self.learning_rate_placeholder: lr\n }), step)\n sys.stdout.flush()\n\n # Update the `TrainStatsLogger`.\n self.train_stats_logger.update(train_eval_iter=step,\n train_loss=l,\n train_error=train_error,\n total_train_error=None,\n val_eval_iter=None,\n val_error=None,\n learning_rate=lr)\n\n # Evaluate model.\n if step % self.eval_frequency == 0 and step > 0:\n if self.cfg[\"eval_total_train_error\"]:\n train_result = self._error_rate_in_batches(\n validation_set=False)\n self.logger.info(\"Training error: {}\".format(\n str(round(train_result.error_rate, 3))))\n\n # Update the `TrainStatsLogger` and save.\n self.train_stats_logger.update(\n train_eval_iter=None,\n train_loss=None,\n train_error=None,\n total_train_error=train_result.error_rate,\n total_train_loss=train_result.cross_entropy_loss,\n val_eval_iter=None,\n val_error=None,\n learning_rate=None)\n self.train_stats_logger.log()\n\n if self.train_pct < 1.0:\n val_result = self._error_rate_in_batches()\n self.summary_writer.add_summary(\n self.sess.run(\n self.merged_eval_summaries,\n feed_dict={\n self.val_error_placeholder: val_result.\n error_rate\n }), step)\n self.logger.info(\"Validation error: {}\".format(\n str(round(val_result.error_rate, 3))))\n self.logger.info(\"Validation loss: {}\".format(\n str(round(val_result.cross_entropy_loss, 3))))\n sys.stdout.flush()\n\n # Update the `TrainStatsLogger`.\n if self.train_pct < 1.0:\n self.train_stats_logger.update(\n train_eval_iter=None,\n train_loss=None,\n train_error=None,\n total_train_error=None,\n val_eval_iter=step,\n val_loss=val_result.cross_entropy_loss,\n val_error=val_result.error_rate,\n learning_rate=None)\n else:\n self.train_stats_logger.update(train_eval_iter=None,\n train_loss=None,\n train_error=None,\n total_train_error=None,\n val_eval_iter=step,\n learning_rate=None)\n\n # Save everything!\n self.train_stats_logger.log()\n\n # Save the model.\n if step % self.save_frequency == 0 and step > 0:\n self.saver.save(\n self.sess,\n os.path.join(self.model_dir,\n GQCNNFilenames.INTER_MODEL.format(step)))\n self.saver.save(\n self.sess,\n os.path.join(self.model_dir,\n GQCNNFilenames.FINAL_MODEL))\n\n # Launch tensorboard only after the first iteration.\n if not self.tensorboard_has_launched:\n self.tensorboard_has_launched = True\n self._launch_tensorboard()\n\n # Get final errors and flush the stdout pipeline.\n final_val_result = self._error_rate_in_batches()\n self.logger.info(\"Final validation error: {}\".format(\n str(round(final_val_result.error_rate, 3))))\n self.logger.info(\"Final validation loss: {}\".format(\n str(round(final_val_result.cross_entropy_loss, 3))))\n if self.cfg[\"eval_total_train_error\"]:\n final_train_result = self._error_rate_in_batches(\n validation_set=False)\n self.logger.info(\"Final training error: {}\".format(\n final_train_result.error_rate))\n self.logger.info(\"Final training loss: {}\".format(\n final_train_result.cross_entropy_loss))\n sys.stdout.flush()\n\n # Update the `TrainStatsLogger`.\n self.train_stats_logger.update(\n train_eval_iter=None,\n train_loss=None,\n train_error=None,\n total_train_error=None,\n val_eval_iter=step,\n val_loss=final_val_result.cross_entropy_loss,\n val_error=final_val_result.error_rate,\n learning_rate=None)\n\n # Log & save everything!\n self.train_stats_logger.log()\n self.saver.save(\n self.sess,\n os.path.join(self.model_dir, GQCNNFilenames.FINAL_MODEL))\n\n except Exception as e:\n self._cleanup()\n raise e\n\n self._cleanup()\n\n def _compute_data_metrics(self):\n \"\"\"Calculate image mean, image std, pose mean, pose std, normalization\n params.\"\"\"\n # Subsample tensors for faster runtime.\n random_file_indices = np.random.choice(self.num_tensors,\n size=self.num_random_files,\n replace=False)\n\n if self.gqcnn.input_depth_mode == InputDepthMode.POSE_STREAM:\n # Compute image stats.\n im_mean_filename = os.path.join(self.model_dir,\n GQCNNFilenames.IM_MEAN)\n im_std_filename = os.path.join(self.model_dir,\n GQCNNFilenames.IM_STD)\n if os.path.exists(im_mean_filename) and os.path.exists(\n im_std_filename):\n self.im_mean = np.load(im_mean_filename)\n self.im_std = np.load(im_std_filename)\n else:\n self.im_mean = 0\n self.im_std = 0\n\n # Compute mean.\n self.logger.info(\"Computing image mean\")\n num_summed = 0\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\n \"Adding file {} of {} to image mean estimate\".\n format(k + 1, random_file_indices.shape[0]))\n im_data = self.dataset.tensor(self.im_field_name, i).arr\n train_indices = self.train_index_map[i]\n if train_indices.shape[0] > 0:\n self.im_mean += np.sum(im_data[train_indices, ...])\n num_summed += self.train_index_map[i].shape[\n 0] * im_data.shape[1] * im_data.shape[2]\n self.im_mean = self.im_mean / num_summed\n\n # Compute std.\n self.logger.info(\"Computing image std\")\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\n \"Adding file {} of {} to image std estimate\".\n format(k + 1, random_file_indices.shape[0]))\n im_data = self.dataset.tensor(self.im_field_name, i).arr\n train_indices = self.train_index_map[i]\n if train_indices.shape[0] > 0:\n self.im_std += np.sum(\n (im_data[train_indices, ...] - self.im_mean)**2)\n self.im_std = np.sqrt(self.im_std / num_summed)\n\n # Save.\n np.save(im_mean_filename, self.im_mean)\n np.save(im_std_filename, self.im_std)\n\n # Update GQ-CNN instance.\n self.gqcnn.set_im_mean(self.im_mean)\n self.gqcnn.set_im_std(self.im_std)\n\n # Compute pose stats.\n pose_mean_filename = os.path.join(self.model_dir,\n GQCNNFilenames.POSE_MEAN)\n pose_std_filename = os.path.join(self.model_dir,\n GQCNNFilenames.POSE_STD)\n if os.path.exists(pose_mean_filename) and os.path.exists(\n pose_std_filename):\n self.pose_mean = np.load(pose_mean_filename)\n self.pose_std = np.load(pose_std_filename)\n else:\n self.pose_mean = np.zeros(self.raw_pose_shape)\n self.pose_std = np.zeros(self.raw_pose_shape)\n\n # Compute mean.\n num_summed = 0\n self.logger.info(\"Computing pose mean\")\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\n \"Adding file {} of {} to pose mean estimate\".\n format(k + 1, random_file_indices.shape[0]))\n pose_data = self.dataset.tensor(self.pose_field_name,\n i).arr\n train_indices = self.train_index_map[i]\n if self.gripper_mode == GripperMode.SUCTION:\n rand_indices = np.random.choice(\n pose_data.shape[0],\n size=pose_data.shape[0] // 2,\n replace=False)\n pose_data[rand_indices,\n 4] = -pose_data[rand_indices, 4]\n elif self.gripper_mode == GripperMode.LEGACY_SUCTION:\n rand_indices = np.random.choice(\n pose_data.shape[0],\n size=pose_data.shape[0] // 2,\n replace=False)\n pose_data[rand_indices,\n 3] = -pose_data[rand_indices, 3]\n if train_indices.shape[0] > 0:\n pose_data = pose_data[train_indices, :]\n pose_data = pose_data[np.isfinite(pose_data[:, 3]), :]\n self.pose_mean += np.sum(pose_data, axis=0)\n num_summed += pose_data.shape[0]\n self.pose_mean = self.pose_mean / num_summed\n\n # Compute std.\n self.logger.info(\"Computing pose std\")\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\n \"Adding file {} of {} to pose std estimate\".format(\n k + 1, random_file_indices.shape[0]))\n pose_data = self.dataset.tensor(self.pose_field_name,\n i).arr\n train_indices = self.train_index_map[i]\n if self.gripper_mode == GripperMode.SUCTION:\n rand_indices = np.random.choice(\n pose_data.shape[0],\n size=pose_data.shape[0] // 2,\n replace=False)\n pose_data[rand_indices,\n 4] = -pose_data[rand_indices, 4]\n elif self.gripper_mode == GripperMode.LEGACY_SUCTION:\n rand_indices = np.random.choice(\n pose_data.shape[0],\n size=pose_data.shape[0] // 2,\n replace=False)\n pose_data[rand_indices,\n 3] = -pose_data[rand_indices, 3]\n if train_indices.shape[0] > 0:\n pose_data = pose_data[train_indices, :]\n pose_data = pose_data[np.isfinite(pose_data[:, 3]), :]\n self.pose_std += np.sum(\n (pose_data - self.pose_mean)**2, axis=0)\n self.pose_std = np.sqrt(self.pose_std / num_summed)\n self.pose_std[self.pose_std == 0] = 1.0\n\n # Save.\n self.pose_mean = read_pose_data(self.pose_mean,\n self.gripper_mode)\n self.pose_std = read_pose_data(self.pose_std,\n self.gripper_mode)\n np.save(pose_mean_filename, self.pose_mean)\n np.save(pose_std_filename, self.pose_std)\n\n # Update GQ-CNN instance.\n self.gqcnn.set_pose_mean(self.pose_mean)\n self.gqcnn.set_pose_std(self.pose_std)\n\n # Check for invalid values.\n if np.any(np.isnan(self.pose_mean)) or np.any(\n np.isnan(self.pose_std)):\n self.logger.error(\n \"Pose mean or pose std is NaN! Check the input dataset\")\n exit(0)\n\n elif self.gqcnn.input_depth_mode == InputDepthMode.SUB:\n # Compute (image - depth) stats.\n im_depth_sub_mean_filename = os.path.join(\n self.model_dir, GQCNNFilenames.IM_DEPTH_SUB_MEAN)\n im_depth_sub_std_filename = os.path.join(\n self.model_dir, GQCNNFilenames.IM_DEPTH_SUB_STD)\n if os.path.exists(im_depth_sub_mean_filename) and os.path.exists(\n im_depth_sub_std_filename):\n self.im_depth_sub_mean = np.load(im_depth_sub_mean_filename)\n self.im_depth_sub_std = np.load(im_depth_sub_std_filename)\n else:\n self.im_depth_sub_mean = 0\n self.im_depth_sub_std = 0\n\n # Compute mean.\n self.logger.info(\"Computing (image - depth) mean.\")\n num_summed = 0\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\"Adding file {} of {}...\".format(\n k + 1, random_file_indices.shape[0]))\n im_data = self.dataset.tensor(self.im_field_name, i).arr\n depth_data = read_pose_data(\n self.dataset.tensor(self.pose_field_name, i).arr,\n self.gripper_mode)\n sub_data = im_data - np.tile(\n np.reshape(depth_data, (-1, 1, 1, 1)),\n (1, im_data.shape[1], im_data.shape[2], 1))\n train_indices = self.train_index_map[i]\n if train_indices.shape[0] > 0:\n self.im_depth_sub_mean += np.sum(\n sub_data[train_indices, ...])\n num_summed += self.train_index_map[i].shape[\n 0] * im_data.shape[1] * im_data.shape[2]\n self.im_depth_sub_mean = self.im_depth_sub_mean / num_summed\n\n # Compute std.\n self.logger.info(\"Computing (image - depth) std.\")\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\"Adding file {} of {}...\".format(\n k + 1, random_file_indices.shape[0]))\n im_data = self.dataset.tensor(self.im_field_name, i).arr\n depth_data = read_pose_data(\n self.dataset.tensor(self.pose_field_name, i).arr,\n self.gripper_mode)\n sub_data = im_data - np.tile(\n np.reshape(depth_data, (-1, 1, 1, 1)),\n (1, im_data.shape[1], im_data.shape[2], 1))\n train_indices = self.train_index_map[i]\n if train_indices.shape[0] > 0:\n self.im_depth_sub_std += np.sum(\n (sub_data[train_indices, ...] -\n self.im_depth_sub_mean)**2)\n self.im_depth_sub_std = np.sqrt(self.im_depth_sub_std /\n num_summed)\n\n # Save.\n np.save(im_depth_sub_mean_filename, self.im_depth_sub_mean)\n np.save(im_depth_sub_std_filename, self.im_depth_sub_std)\n\n # Update GQ-CNN instance.\n self.gqcnn.set_im_depth_sub_mean(self.im_depth_sub_mean)\n self.gqcnn.set_im_depth_sub_std(self.im_depth_sub_std)\n\n elif self.gqcnn.input_depth_mode == InputDepthMode.IM_ONLY:\n # Compute image stats.\n im_mean_filename = os.path.join(self.model_dir,\n GQCNNFilenames.IM_MEAN)\n im_std_filename = os.path.join(self.model_dir,\n GQCNNFilenames.IM_STD)\n if os.path.exists(im_mean_filename) and os.path.exists(\n im_std_filename):\n self.im_mean = np.load(im_mean_filename)\n self.im_std = np.load(im_std_filename)\n else:\n self.im_mean = 0\n self.im_std = 0\n\n # Compute mean.\n self.logger.info(\"Computing image mean.\")\n num_summed = 0\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\"Adding file {} of {}...\".format(\n k + 1, random_file_indices.shape[0]))\n im_data = self.dataset.tensor(self.im_field_name, i).arr\n train_indices = self.train_index_map[i]\n if train_indices.shape[0] > 0:\n self.im_mean += np.sum(im_data[train_indices, ...])\n num_summed += self.train_index_map[i].shape[\n 0] * im_data.shape[1] * im_data.shape[2]\n self.im_mean = self.im_mean / num_summed\n\n # Compute std.\n self.logger.info(\"Computing image std.\")\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\"Adding file {} of {}...\".format(\n k + 1, random_file_indices.shape[0]))\n im_data = self.dataset.tensor(self.im_field_name, i).arr\n train_indices = self.train_index_map[i]\n if train_indices.shape[0] > 0:\n self.im_std += np.sum(\n (im_data[train_indices, ...] - self.im_mean)**2)\n self.im_std = np.sqrt(self.im_std / num_summed)\n\n # Save.\n np.save(im_mean_filename, self.im_mean)\n np.save(im_std_filename, self.im_std)\n\n # Update GQ-CNN instance.\n self.gqcnn.set_im_mean(self.im_mean)\n self.gqcnn.set_im_std(self.im_std)\n\n # Compute normalization parameters of the network.\n pct_pos_train_filename = os.path.join(self.model_dir,\n GQCNNFilenames.PCT_POS_TRAIN)\n pct_pos_val_filename = os.path.join(self.model_dir,\n GQCNNFilenames.PCT_POS_VAL)\n if os.path.exists(pct_pos_train_filename) and os.path.exists(\n pct_pos_val_filename):\n pct_pos_train = np.load(pct_pos_train_filename)\n pct_pos_val = np.load(pct_pos_val_filename)\n else:\n self.logger.info(\"Computing grasp quality metric stats.\")\n all_train_metrics = None\n all_val_metrics = None\n\n # Read metrics.\n for k, i in enumerate(random_file_indices):\n if k % self.preproc_log_frequency == 0:\n self.logger.info(\"Adding file {} of {}...\".format(\n k + 1, random_file_indices.shape[0]))\n metric_data = self.dataset.tensor(self.label_field_name, i).arr\n train_indices = self.train_index_map[i]\n val_indices = self.val_index_map[i]\n\n if train_indices.shape[0] > 0:\n train_metric_data = metric_data[train_indices]\n if all_train_metrics is None:\n all_train_metrics = train_metric_data\n else:\n all_train_metrics = np.r_[all_train_metrics,\n train_metric_data]\n\n if val_indices.shape[0] > 0:\n val_metric_data = metric_data[val_indices]\n if all_val_metrics is None:\n all_val_metrics = val_metric_data\n else:\n all_val_metrics = np.r_[all_val_metrics,\n val_metric_data]\n\n # Compute train stats.\n self.min_metric = np.min(all_train_metrics)\n self.max_metric = np.max(all_train_metrics)\n self.mean_metric = np.mean(all_train_metrics)\n self.median_metric = np.median(all_train_metrics)\n\n # Save metrics.\n pct_pos_train = np.sum(all_train_metrics > self.metric_thresh\n ) / all_train_metrics.shape[0]\n np.save(pct_pos_train_filename, np.array(pct_pos_train))\n\n if self.train_pct < 1.0:\n pct_pos_val = np.sum(all_val_metrics > self.metric_thresh\n ) / all_val_metrics.shape[0]\n np.save(pct_pos_val_filename, np.array(pct_pos_val))\n\n self.logger.info(\"Percent positive in train: \" + str(pct_pos_train))\n if self.train_pct < 1.0:\n self.logger.info(\"Percent positive in val: \" + str(pct_pos_val))\n\n if self._angular_bins > 0:\n self.logger.info(\"Calculating angular bin statistics...\")\n bin_counts = np.zeros((self._angular_bins, ))\n for m in range(self.num_tensors):\n pose_arr = self.dataset.tensor(self.pose_field_name, m).arr\n angles = pose_arr[:, 3]\n neg_ind = np.where(angles < 0)\n angles = np.abs(angles) % self._max_angle\n angles[neg_ind] *= -1\n g_90 = np.where(angles > (self._max_angle / 2))\n l_neg_90 = np.where(angles < (-1 * (self._max_angle / 2)))\n angles[g_90] -= self._max_angle\n angles[l_neg_90] += self._max_angle\n # TODO(vsatish): Actually fix this.\n angles *= -1 # Hack to fix reverse angle convention.\n angles += (self._max_angle / 2)\n for i in range(angles.shape[0]):\n bin_counts[int(angles[i] // self._bin_width)] += 1\n self.logger.info(\"Bin counts: {}.\".format(bin_counts))\n\n def _compute_split_indices(self):\n \"\"\"Compute train and validation indices for each tensor to speed data\n accesses.\"\"\"\n # Read indices.\n train_indices, val_indices, _ = self.dataset.split(self.split_name)\n\n # Loop through tensors, assigning indices to each file.\n self.train_index_map = {}\n for i in range(self.dataset.num_tensors):\n self.train_index_map[i] = []\n\n for i in train_indices:\n tensor_index = self.dataset.tensor_index(i)\n datapoint_indices = self.dataset.datapoint_indices_for_tensor(\n tensor_index)\n lowest = np.min(datapoint_indices)\n self.train_index_map[tensor_index].append(i - lowest)\n\n for i, indices in self.train_index_map.items():\n self.train_index_map[i] = np.array(indices)\n\n self.val_index_map = {}\n for i in range(self.dataset.num_tensors):\n self.val_index_map[i] = []\n\n for i in val_indices:\n tensor_index = self.dataset.tensor_index(i)\n if tensor_index not in self.val_index_map:\n self.val_index_map[tensor_index] = []\n datapoint_indices = self.dataset.datapoint_indices_for_tensor(\n tensor_index)\n lowest = np.min(datapoint_indices)\n self.val_index_map[tensor_index].append(i - lowest)\n\n for i, indices in self.val_index_map.items():\n self.val_index_map[i] = np.array(indices)\n\n def _setup_output_dirs(self):\n \"\"\"Setup output directories.\"\"\"\n self.logger.info(\"Saving model to: {}\".format(self.model_dir))\n\n # Create the summary dir.\n self.summary_dir = os.path.join(self.model_dir,\n \"tensorboard_summaries\")\n if not os.path.exists(self.summary_dir):\n os.mkdir(self.summary_dir)\n else:\n # If the summary directory already exists, clean it out by deleting\n # all files in it. We don't want Tensorboard to get confused with\n # old logs while reusing the same directory.\n old_files = os.listdir(self.summary_dir)\n for f in old_files:\n os.remove(os.path.join(self.summary_dir, f))\n\n # Setup filter directory.\n self.filter_dir = os.path.join(self.model_dir, \"filters\")\n if not os.path.exists(self.filter_dir):\n os.mkdir(self.filter_dir)\n\n def _save_configs(self):\n \"\"\"Save training configuration.\"\"\"\n # Update config for fine-tuning.\n if self.finetuning:\n self.cfg[\"base_model_dir\"] = self.base_model_dir\n\n # Save config.\n out_config_filename = os.path.join(self.model_dir,\n GQCNNFilenames.SAVED_CFG)\n tempOrderedDict = collections.OrderedDict()\n for key in self.cfg:\n tempOrderedDict[key] = self.cfg[key]\n with open(out_config_filename, \"w\") as outfile:\n json.dump(tempOrderedDict, outfile, indent=JSON_INDENT)\n\n # Save training script.\n this_filename = sys.argv[0]\n out_train_filename = os.path.join(self.model_dir, \"training_script.py\")\n shutil.copyfile(this_filename, out_train_filename)\n\n # Save architecture.\n out_architecture_filename = os.path.join(self.model_dir,\n GQCNNFilenames.SAVED_ARCH)\n json.dump(self.cfg[\"gqcnn\"][\"architecture\"],\n open(out_architecture_filename, \"w\"),\n indent=JSON_INDENT)\n\n def _read_training_params(self):\n \"\"\"Read training parameters from configuration file.\"\"\"\n # Splits.\n self.train_pct = self.cfg[\"train_pct\"]\n self.total_pct = self.cfg[\"total_pct\"]\n\n # Training sizes.\n self.train_batch_size = self.cfg[\"train_batch_size\"]\n self.val_batch_size = self.cfg[\"val_batch_size\"]\n self.max_files_eval = None\n if \"max_files_eval\" in self.cfg:\n self.max_files_eval = self.cfg[\"max_files_eval\"]\n\n # Logging.\n self.num_epochs = self.cfg[\"num_epochs\"]\n self.eval_frequency = self.cfg[\"eval_frequency\"]\n self.save_frequency = self.cfg[\"save_frequency\"]\n self.log_frequency = self.cfg[\"log_frequency\"]\n\n # Optimization.\n self.train_l2_regularizer = self.cfg[\"train_l2_regularizer\"]\n self.base_lr = self.cfg[\"base_lr\"]\n self.decay_step_multiplier = self.cfg[\"decay_step_multiplier\"]\n self.decay_rate = self.cfg[\"decay_rate\"]\n self.momentum_rate = self.cfg[\"momentum_rate\"]\n self.max_training_examples_per_load = self.cfg[\n \"max_training_examples_per_load\"]\n self.drop_rate = self.cfg[\"drop_rate\"]\n self.max_global_grad_norm = self.cfg[\"max_global_grad_norm\"]\n self.optimize_base_layers = False\n if \"optimize_base_layers\" in self.cfg:\n self.optimize_base_layers = self.cfg[\"optimize_base_layers\"]\n\n # Metrics.\n self.target_metric_name = self.cfg[\"target_metric_name\"]\n self.metric_thresh = self.cfg[\"metric_thresh\"]\n self.training_mode = self.cfg[\"training_mode\"]\n if self.training_mode != TrainingMode.CLASSIFICATION:\n raise ValueError(\n \"Training mode '{}' not currently supported!\".format(\n self.training_mode))\n\n # Tensorboad.\n self._tensorboard_port = self.cfg[\"tensorboard_port\"]\n\n # Preprocessing.\n self.preproc_log_frequency = self.cfg[\"preproc_log_frequency\"]\n self.num_random_files = self.cfg[\"num_random_files\"]\n self.max_prefetch_q_size = GeneralConstants.MAX_PREFETCH_Q_SIZE\n if \"max_prefetch_q_size\" in self.cfg:\n self.max_prefetch_q_size = self.cfg[\"max_prefetch_q_size\"]\n self.num_prefetch_q_workers = GeneralConstants.NUM_PREFETCH_Q_WORKERS\n if \"num_prefetch_q_workers\" in self.cfg:\n self.num_prefetch_q_workers = self.cfg[\"num_prefetch_q_workers\"]\n\n # Re-weighting positives/negatives.\n self.pos_weight = 0.0\n if \"pos_weight\" in self.cfg:\n self.pos_weight = self.cfg[\"pos_weight\"]\n self.pos_accept_prob = 1.0\n self.neg_accept_prob = 1.0\n if self.pos_weight > 1:\n self.neg_accept_prob = 1 / self.pos_weight\n else:\n self.pos_accept_prob = self.pos_weight\n\n if self.train_pct < 0 or self.train_pct > 1:\n raise ValueError(\"Train percentage must be in range [0,1]\")\n\n if self.total_pct < 0 or self.total_pct > 1:\n raise ValueError(\"Total percentage must be in range [0,1]\")\n\n # Input normalization.\n self._norm_inputs = True\n if self.gqcnn.input_depth_mode == InputDepthMode.SUB:\n self._norm_inputs = False\n\n # Angular training.\n self._angular_bins = self.gqcnn.angular_bins\n self._max_angle = self.gqcnn.max_angle\n\n # During angular training, make sure symmetrization in denoising is\n # turned off and also set the angular bin width.\n if self._angular_bins > 0:\n symmetrization_msg = (\"Symmetrization denoising must be turned off\"\n \" during angular training.\")\n assert not self.cfg[\"symmetrize\"], symmetrization_msg\n self._bin_width = self._max_angle / self._angular_bins\n\n # Debugging.\n self._debug = self.cfg[\"debug\"]\n self._seed = self.cfg[\"seed\"]\n if self._debug:\n if self.num_prefetch_q_workers > 1:\n self.logger.warning(\n \"Deterministic execution is not possible with \"\n \"more than one prefetch queue worker even in debug mode.\")\n # This reduces initialization time for fast debugging.\n self.num_random_files = self.cfg[\"debug_num_files\"]\n\n np.random.seed(self._seed)\n random.seed(self._seed)\n\n def _setup_denoising_and_synthetic(self):\n \"\"\"Setup denoising and synthetic data parameters.\"\"\"\n # Multiplicative denoising.\n if self.cfg[\"multiplicative_denoising\"]:\n self.gamma_shape = self.cfg[\"gamma_shape\"]\n self.gamma_scale = 1 / self.gamma_shape\n\n # Gaussian process noise.\n if self.cfg[\"gaussian_process_denoising\"]:\n self.gp_rescale_factor = self.cfg[\n \"gaussian_process_scaling_factor\"]\n self.gp_sample_height = int(self.im_height /\n self.gp_rescale_factor)\n self.gp_sample_width = int(self.im_width / self.gp_rescale_factor)\n self.gp_num_pix = self.gp_sample_height * self.gp_sample_width\n self.gp_sigma = self.cfg[\"gaussian_process_sigma\"]\n\n def _open_dataset(self):\n \"\"\"Open the dataset.\"\"\"\n # Read in filenames of training data (poses, images, labels).\n self.dataset = TensorDataset.open(self.dataset_dir)\n self.num_datapoints = self.dataset.num_datapoints\n self.num_tensors = self.dataset.num_tensors\n self.datapoints_per_file = self.dataset.datapoints_per_file\n self.num_random_files = min(self.num_tensors, self.num_random_files)\n\n # Read split.\n if not self.dataset.has_split(self.split_name):\n self.logger.info(\n \"Dataset split: {} not found. Creating new split...\".format(\n self.split_name))\n self.dataset.make_split(self.split_name, train_pct=self.train_pct)\n else:\n self.logger.info(\"Training split: {} found in dataset.\".format(\n self.split_name))\n self._compute_split_indices()\n\n def _compute_data_params(self):\n \"\"\"Compute parameters of the dataset.\"\"\"\n # Image params.\n self.im_field_name = self.cfg[\"image_field_name\"]\n self.im_height = self.dataset.config[\"fields\"][\n self.im_field_name][\"height\"]\n self.im_width = self.dataset.config[\"fields\"][\n self.im_field_name][\"width\"]\n self.im_channels = self.dataset.config[\"fields\"][\n self.im_field_name][\"channels\"]\n # NOTE: There was originally some weird math going on here...\n self.im_center = np.array([self.im_height // 2, self.im_width // 2])\n\n # Poses.\n self.pose_field_name = self.cfg[\"pose_field_name\"]\n self.gripper_mode = self.gqcnn.gripper_mode\n self.pose_dim = pose_dim(self.gripper_mode)\n self.raw_pose_shape = self.dataset.config[\"fields\"][\n self.pose_field_name][\"height\"]\n\n # Outputs.\n self.label_field_name = self.target_metric_name\n self.num_categories = 2\n\n # Compute the number of train and val examples.\n self.num_train = 0\n self.num_val = 0\n for train_indices in self.train_index_map.values():\n self.num_train += train_indices.shape[0]\n for val_indices in self.train_index_map.values():\n self.num_val += val_indices.shape[0]\n\n # Set params based on the number of training examples (convert epochs\n # to steps).\n self.eval_frequency = int(\n np.ceil(self.eval_frequency *\n (self.num_train / self.train_batch_size)))\n self.save_frequency = int(\n np.ceil(self.save_frequency *\n (self.num_train / self.train_batch_size)))\n self.decay_step = self.decay_step_multiplier * self.num_train\n\n def _setup_tensorflow(self):\n \"\"\"Setup Tensorflow placeholders, session, and queue.\"\"\"\n\n # Set up training label and NumPy data types.\n if self.training_mode == TrainingMode.REGRESSION:\n train_label_dtype = tf.float32\n self.numpy_dtype = np.float32\n elif self.training_mode == TrainingMode.CLASSIFICATION:\n train_label_dtype = tf.int64\n self.numpy_dtype = np.int64\n if self.cfg[\"loss\"] == \"weighted_cross_entropy\":\n train_label_dtype = tf.float32\n self.numpy_dtype = np.float32\n else:\n raise ValueError(\"Training mode '{}' not supported\".format(\n self.training_mode))\n\n # Set up placeholders.\n self.train_labels_node = tf.placeholder(train_label_dtype,\n (self.train_batch_size, ))\n self.input_im_node = tf.placeholder(\n tf.float32, (self.train_batch_size, self.im_height, self.im_width,\n self.im_channels))\n self.input_pose_node = tf.placeholder(\n tf.float32, (self.train_batch_size, self.pose_dim))\n if self._angular_bins > 0:\n self.train_pred_mask_node = tf.placeholder(\n tf.int32, (self.train_batch_size, self._angular_bins * 2))\n\n # Create data prefetch queue.\n self.prefetch_q = mp.Queue(self.max_prefetch_q_size)\n\n # Get weights.\n self.weights = self.gqcnn.weights\n\n # Open a TF session for the GQ-CNN instance and store it also as the\n # optimizer session.\n self.sess = self.gqcnn.open_session()\n\n # Setup data prefetch queue worker termination event.\n self.term_event = mp.Event()\n self.term_event.clear()\n\n def _setup_summaries(self):\n \"\"\"Sets up placeholders for summary values and creates summary\n writer.\"\"\"\n # Create placeholders for Python values because `tf.summary.scalar`\n # expects a placeholder.\n self.val_error_placeholder = tf.placeholder(tf.float32, [])\n self.minibatch_error_placeholder = tf.placeholder(tf.float32, [])\n self.minibatch_loss_placeholder = tf.placeholder(tf.float32, [])\n self.learning_rate_placeholder = tf.placeholder(tf.float32, [])\n\n # Tag the `tf.summary.scalar`s so that we can group them together and\n # write different batches of summaries at different intervals.\n tf.summary.scalar(\"val_error\",\n self.val_error_placeholder,\n collections=[\"eval_frequency\"])\n tf.summary.scalar(\"minibatch_error\",\n self.minibatch_error_placeholder,\n collections=[\"log_frequency\"])\n tf.summary.scalar(\"minibatch_loss\",\n self.minibatch_loss_placeholder,\n collections=[\"log_frequency\"])\n tf.summary.scalar(\"learning_rate\",\n self.learning_rate_placeholder,\n collections=[\"log_frequency\"])\n self.merged_eval_summaries = tf.summary.merge_all(\"eval_frequency\")\n self.merged_log_summaries = tf.summary.merge_all(\"log_frequency\")\n\n # Create a TF summary writer with the specified summary directory.\n self.summary_writer = tf.summary.FileWriter(self.summary_dir)\n\n # Initialize the variables again now that we have added some new ones.\n with self.sess.as_default():\n tf.global_variables_initializer().run()\n\n def _cleanup(self):\n self.logger.info(\"Cleaning and preparing to exit optimization...\")\n\n # Set termination event for prefetch queue workers.\n self.logger.info(\"Terminating prefetch queue workers...\")\n self.term_event.set()\n\n # Flush prefetch queue.\n # NOTE: This prevents a deadlock with the worker process queue buffers.\n self._flush_prefetch_queue()\n\n # Join prefetch queue worker processes.\n for p in self.prefetch_q_workers:\n p.join()\n\n # Close tensorboard if started.\n if self.tensorboard_has_launched:\n self._close_tensorboard()\n\n # Close Tensorflow session.\n self.gqcnn.close_session()\n\n def _flush_prefetch_queue(self):\n \"\"\"Flush prefetch queue.\"\"\"\n self.logger.info(\"Flushing prefetch queue...\")\n for i in range(self.prefetch_q.qsize()):\n self.prefetch_q.get()\n\n def _setup(self):\n \"\"\"Setup for training.\"\"\"\n # Initialize data prefetch queue thread exit booleans.\n self.queue_thread_exited = False\n self.forceful_exit = False\n\n # Setup output directories.\n self._setup_output_dirs()\n\n # Save training configuration.\n self._save_configs()\n\n # Read training parameters from config file.\n self._read_training_params()\n\n # Setup image and pose data files.\n self._open_dataset()\n\n # Compute data parameters.\n self._compute_data_params()\n\n # Setup denoising and synthetic data parameters.\n self._setup_denoising_and_synthetic()\n\n # Compute means, std's, and normalization metrics.\n self._compute_data_metrics()\n\n # Setup Tensorflow session/placeholders/queue.\n self._setup_tensorflow()\n\n # Setup summaries for visualizing metrics in Tensorboard.\n self._setup_summaries()\n\n def _load_and_enqueue(self, seed):\n \"\"\"Loads and enqueues a batch of images for training.\"\"\"\n\n # When the parent process receives a SIGINT, it will itself handle\n # cleaning up child processes.\n signal.signal(signal.SIGINT, signal.SIG_IGN)\n\n # Set the random seed explicitly to prevent all workers from possible\n # inheriting the same seed on process initialization.\n np.random.seed(seed)\n random.seed(seed)\n\n # Open dataset.\n dataset = TensorDataset.open(self.dataset_dir)\n\n while not self.term_event.is_set():\n # Loop through data.\n num_queued = 0\n start_i = 0\n end_i = 0\n file_num = 0\n queue_start = time.time()\n\n # Init buffers.\n train_images = np.zeros([\n self.train_batch_size, self.im_height, self.im_width,\n self.im_channels\n ]).astype(np.float32)\n train_poses = np.zeros([self.train_batch_size,\n self.pose_dim]).astype(np.float32)\n train_labels = np.zeros(self.train_batch_size).astype(\n self.numpy_dtype)\n if self._angular_bins > 0:\n train_pred_mask = np.zeros(\n (self.train_batch_size, self._angular_bins * 2),\n dtype=bool)\n\n while start_i < self.train_batch_size:\n # Compute num remaining.\n num_remaining = self.train_batch_size - num_queued\n\n # Generate tensor index uniformly at random.\n file_num = np.random.choice(self.num_tensors, size=1)[0]\n\n read_start = time.time()\n train_images_tensor = dataset.tensor(self.im_field_name,\n file_num)\n train_poses_tensor = dataset.tensor(self.pose_field_name,\n file_num)\n train_labels_tensor = dataset.tensor(self.label_field_name,\n file_num)\n read_stop = time.time()\n self.logger.debug(\"Reading data took {} sec\".format(\n str(round(read_stop - read_start, 3))))\n self.logger.debug(\"File num: {}\".format(file_num))\n\n # Get batch indices uniformly at random.\n train_ind = self.train_index_map[file_num]\n np.random.shuffle(train_ind)\n if self.gripper_mode == GripperMode.LEGACY_SUCTION:\n tp_tmp = read_pose_data(train_poses_tensor.data,\n self.gripper_mode)\n train_ind = train_ind[np.isfinite(tp_tmp[train_ind, 1])]\n\n # Filter positives and negatives.\n if self.training_mode == TrainingMode.CLASSIFICATION and \\\n self.pos_weight != 0.0:\n labels = 1 * (train_labels_tensor.arr > self.metric_thresh)\n np.random.shuffle(train_ind)\n filtered_ind = []\n for index in train_ind:\n if labels[index] == 0 and np.random.rand(\n ) < self.neg_accept_prob:\n filtered_ind.append(index)\n elif labels[index] == 1 and np.random.rand(\n ) < self.pos_accept_prob:\n filtered_ind.append(index)\n train_ind = np.array(filtered_ind)\n\n # Samples train indices.\n upper = min(num_remaining, train_ind.shape[0],\n self.max_training_examples_per_load)\n ind = train_ind[:upper]\n num_loaded = ind.shape[0]\n if num_loaded == 0:\n self.logger.warning(\"Queueing zero examples!!!!\")\n continue\n\n # Subsample data.\n train_images_arr = train_images_tensor.arr[ind, ...]\n train_poses_arr = train_poses_tensor.arr[ind, ...]\n angles = train_poses_arr[:, 3]\n train_label_arr = train_labels_tensor.arr[ind]\n num_images = train_images_arr.shape[0]\n\n # Resize images.\n rescale_factor = self.im_height / train_images_arr.shape[1]\n if rescale_factor != 1.0:\n resized_train_images_arr = np.zeros([\n num_images, self.im_height, self.im_width,\n self.im_channels\n ]).astype(np.float32)\n for i in range(num_images):\n for c in range(train_images_arr.shape[3]):\n resized_train_images_arr[i, :, :, c] = imresize(\n train_images_arr[i, :, :, c],\n rescale_factor,\n interp=\"bicubic\")\n train_images_arr = resized_train_images_arr\n\n # Add noises to images.\n train_images_arr, train_poses_arr = self._distort(\n train_images_arr, train_poses_arr)\n\n # Slice poses.\n train_poses_arr = read_pose_data(train_poses_arr,\n self.gripper_mode)\n\n # Standardize inputs and outputs.\n if self._norm_inputs:\n train_images_arr = (train_images_arr -\n self.im_mean) / self.im_std\n if self.gqcnn.input_depth_mode == \\\n InputDepthMode.POSE_STREAM:\n train_poses_arr = (train_poses_arr -\n self.pose_mean) / self.pose_std\n train_label_arr = 1 * (train_label_arr > self.metric_thresh)\n train_label_arr = train_label_arr.astype(self.numpy_dtype)\n\n if self._angular_bins > 0:\n bins = np.zeros_like(train_label_arr)\n # Form prediction mask to use when calculating loss.\n neg_ind = np.where(angles < 0)\n angles = np.abs(angles) % self._max_angle\n angles[neg_ind] *= -1\n g_90 = np.where(angles > (self._max_angle / 2))\n l_neg_90 = np.where(angles < (-1 * (self._max_angle / 2)))\n angles[g_90] -= self._max_angle\n angles[l_neg_90] += self._max_angle\n # TODO(vsatish): Actually fix this.\n angles *= -1 # Hack to fix reverse angle convention.\n angles += (self._max_angle / 2)\n train_pred_mask_arr = np.zeros(\n (train_label_arr.shape[0], self._angular_bins * 2))\n for i in range(angles.shape[0]):\n bins[i] = angles[i] // self._bin_width\n train_pred_mask_arr[i,\n int((angles[i] //\n self._bin_width) * 2)] = 1\n train_pred_mask_arr[\n i, int((angles[i] // self._bin_width) * 2 + 1)] = 1\n\n # Compute the number of examples loaded.\n num_loaded = train_images_arr.shape[0]\n end_i = start_i + num_loaded\n\n # Enqueue training data batch.\n train_images[start_i:end_i, ...] = train_images_arr.copy()\n train_poses[start_i:end_i, :] = train_poses_arr.copy()\n train_labels[start_i:end_i] = train_label_arr.copy()\n if self._angular_bins > 0:\n train_pred_mask[start_i:end_i] = train_pred_mask_arr.copy()\n\n # Update start index.\n start_i = end_i\n num_queued += num_loaded\n\n # Send data to queue.\n if not self.term_event.is_set():\n try:\n if self._angular_bins > 0:\n self.prefetch_q.put_nowait(\n (train_images, train_poses, train_labels,\n train_pred_mask))\n else:\n self.prefetch_q.put_nowait(\n (train_images, train_poses, train_labels))\n except Queue.Full:\n time.sleep(GeneralConstants.QUEUE_SLEEP)\n queue_stop = time.time()\n self.logger.debug(\"Queue batch took {} sec\".format(\n str(round(queue_stop - queue_start, 3))))\n\n def _distort(self, image_arr, pose_arr):\n \"\"\"Adds noise to a batch of images.\"\"\"\n # Read params.\n num_images = image_arr.shape[0]\n\n # Denoising and synthetic data generation.\n if self.cfg[\"multiplicative_denoising\"]:\n mult_samples = ss.gamma.rvs(self.gamma_shape,\n scale=self.gamma_scale,\n size=num_images)\n mult_samples = mult_samples[:, np.newaxis, np.newaxis, np.newaxis]\n image_arr = image_arr * np.tile(\n mult_samples,\n [1, self.im_height, self.im_width, self.im_channels])\n\n # Add correlated Gaussian noise.\n if self.cfg[\"gaussian_process_denoising\"]:\n for i in range(num_images):\n if np.random.rand() < self.cfg[\"gaussian_process_rate\"]:\n train_image = image_arr[i, :, :, 0]\n gp_noise = ss.norm.rvs(scale=self.gp_sigma,\n size=self.gp_num_pix).reshape(\n self.gp_sample_height,\n self.gp_sample_width)\n gp_noise = imresize(gp_noise,\n self.gp_rescale_factor,\n interp=\"bicubic\")\n train_image[train_image > 0] += gp_noise[train_image > 0]\n image_arr[i, :, :, 0] = train_image\n\n # Symmetrize images.\n if self.cfg[\"symmetrize\"]:\n for i in range(num_images):\n train_image = image_arr[i, :, :, 0]\n # Rotate with 50% probability.\n if np.random.rand() < 0.5:\n theta = 180.0\n rot_map = cv2.getRotationMatrix2D(tuple(self.im_center),\n theta, 1)\n train_image = cv2.warpAffine(\n train_image,\n rot_map, (self.im_height, self.im_width),\n flags=cv2.INTER_NEAREST)\n\n if self.gripper_mode == GripperMode.LEGACY_SUCTION:\n pose_arr[:, 3] = -pose_arr[:, 3]\n elif self.gripper_mode == GripperMode.SUCTION:\n pose_arr[:, 4] = -pose_arr[:, 4]\n # Reflect left right with 50% probability.\n if np.random.rand() < 0.5:\n train_image = np.fliplr(train_image)\n # Reflect up down with 50% probability.\n if np.random.rand() < 0.5:\n train_image = np.flipud(train_image)\n\n if self.gripper_mode == GripperMode.LEGACY_SUCTION:\n pose_arr[:, 3] = -pose_arr[:, 3]\n elif self.gripper_mode == GripperMode.SUCTION:\n pose_arr[:, 4] = -pose_arr[:, 4]\n image_arr[i, :, :, 0] = train_image\n return image_arr, pose_arr\n\n def _error_rate_in_batches(self, num_files_eval=None, validation_set=True):\n \"\"\"Compute error and loss over either training or validation set.\n\n Returns\n -------\n :obj:\"autolab_core.BinaryClassificationResult`\n validation error\n \"\"\"\n all_predictions = []\n all_labels = []\n\n # Subsample files.\n file_indices = np.arange(self.num_tensors)\n if num_files_eval is None:\n num_files_eval = self.max_files_eval\n np.random.shuffle(file_indices)\n if self.max_files_eval is not None and num_files_eval > 0:\n file_indices = file_indices[:num_files_eval]\n\n for i in file_indices:\n # Load next file.\n images = self.dataset.tensor(self.im_field_name, i).arr\n poses = self.dataset.tensor(self.pose_field_name, i).arr\n raw_poses = np.array(poses, copy=True)\n labels = self.dataset.tensor(self.label_field_name, i).arr\n\n # If no datapoints from this file are in validation then just\n # continue.\n if validation_set:\n indices = self.val_index_map[i]\n else:\n indices = self.train_index_map[i]\n if len(indices) == 0:\n continue\n\n images = images[indices, ...]\n poses = read_pose_data(poses[indices, :], self.gripper_mode)\n raw_poses = raw_poses[indices, :]\n labels = labels[indices]\n\n if self.training_mode == TrainingMode.CLASSIFICATION:\n labels = 1 * (labels > self.metric_thresh)\n labels = labels.astype(np.uint8)\n\n if self._angular_bins > 0:\n # Form mask to extract predictions from ground-truth angular\n # bins.\n angles = raw_poses[:, 3]\n neg_ind = np.where(angles < 0)\n angles = np.abs(angles) % self._max_angle\n angles[neg_ind] *= -1\n g_90 = np.where(angles > (self._max_angle / 2))\n l_neg_90 = np.where(angles < (-1 * (self._max_angle / 2)))\n angles[g_90] -= self._max_angle\n angles[l_neg_90] += self._max_angle\n # TODO(vsatish): Actually fix this.\n angles *= -1 # Hack to fix reverse angle convention.\n angles += (self._max_angle / 2)\n pred_mask = np.zeros((labels.shape[0], self._angular_bins * 2),\n dtype=bool)\n for i in range(angles.shape[0]):\n pred_mask[i, int(\n (angles[i] // self._bin_width) * 2)] = True\n pred_mask[i,\n int((angles[i] // self._bin_width) * 2 +\n 1)] = True\n\n # Get predictions.\n predictions = self.gqcnn.predict(images, poses)\n\n if self._angular_bins > 0:\n predictions = predictions[pred_mask].reshape((-1, 2))\n\n # Update.\n all_predictions.extend(predictions[:, 1].tolist())\n all_labels.extend(labels.tolist())\n\n # Get learning result.\n result = None\n if self.training_mode == TrainingMode.CLASSIFICATION:\n result = BinaryClassificationResult(all_predictions, all_labels)\n else:\n result = RegressionResult(all_predictions, all_labels)\n return result\n" }, { "path": "gqcnn/utils/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\"\"\"\nfrom .enums import (ImageMode, TrainingMode, GripperMode, InputDepthMode,\n GeneralConstants, GQCNNTrainingStatus, GQCNNFilenames)\nfrom .policy_exceptions import (NoValidGraspsException,\n NoAntipodalPairsFoundException)\nfrom .train_stats_logger import TrainStatsLogger\nfrom .utils import (is_py2, set_cuda_visible_devices, pose_dim, read_pose_data,\n reduce_shape, weight_name_to_layer_name, imresize)\n\n__all__ = [\n \"is_py2\", \"set_cuda_visible_devices\", \"pose_dim\", \"read_pose_data\",\n \"reduce_shape\", \"weight_name_to_layer_name\", \"imresize\", \"ImageMode\",\n \"TrainingMode\", \"GripperMode\", \"InputDepthMode\", \"GeneralConstants\",\n \"GQCNNTrainingStatus\", \"NoValidGraspsException\",\n \"NoAntipodalPairsFoundException\", \"TrainStatsLogger\", \"GQCNNFilenames\"\n]\n" }, { "path": "gqcnn/utils/enums.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nConstants/enums.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nimport math\n\nimport tensorflow as tf\n\n\n# Other constants.\nclass GeneralConstants(object):\n SEED = 3472134\n SEED_SAMPLE_MAX = 2**32 - 1 # Max range for `np.random.seed`.\n timeout_option = tf.RunOptions(timeout_in_ms=1000000)\n MAX_PREFETCH_Q_SIZE = 250\n NUM_PREFETCH_Q_WORKERS = 3\n QUEUE_SLEEP = 0.001\n PI = math.pi\n FIGSIZE = 16 # For visualization.\n\n\n# Enum for image modalities.\nclass ImageMode(object):\n BINARY = \"binary\"\n DEPTH = \"depth\"\n BINARY_TF = \"binary_tf\"\n COLOR_TF = \"color_tf\"\n GRAY_TF = \"gray_tf\"\n DEPTH_TF = \"depth_tf\"\n DEPTH_TF_TABLE = \"depth_tf_table\"\n TF_DEPTH_IMS = \"tf_depth_ims\"\n\n\n# Enum for training modes.\nclass TrainingMode(object):\n CLASSIFICATION = \"classification\"\n REGRESSION = \"regression\" # Has not been tested, for experimentation only!\n\n\n# Enum for input pose data formats.\nclass GripperMode(object):\n PARALLEL_JAW = \"parallel_jaw\"\n SUCTION = \"suction\"\n MULTI_SUCTION = \"multi_suction\"\n LEGACY_PARALLEL_JAW = \"legacy_parallel_jaw\"\n LEGACY_SUCTION = \"legacy_suction\"\n\n\n# Enum for input depth mode.\nclass InputDepthMode(object):\n POSE_STREAM = \"pose_stream\"\n SUB = \"im_depth_sub\"\n IM_ONLY = \"im_only\"\n\n\n# Enum for training status.\nclass GQCNNTrainingStatus(object):\n NOT_STARTED = \"not_started\"\n SETTING_UP = \"setting_up\"\n TRAINING = \"training\"\n\n\n# Enum for filenames.\nclass GQCNNFilenames(object):\n PCT_POS_VAL = \"pct_pos_val.npy\"\n PCT_POS_TRAIN = \"pct_pos_train.npy\"\n LEARNING_RATES = \"learning_rates.npy\"\n\n TRAIN_ITERS = \"train_eval_iters.npy\"\n TRAIN_LOSSES = \"train_losses.npy\"\n TRAIN_ERRORS = \"train_errors.npy\"\n TOTAL_TRAIN_LOSSES = \"total_train_losses.npy\"\n TOTAL_TRAIN_ERRORS = \"total_train_errors.npy\"\n\n VAL_ITERS = \"val_eval_iters.npy\"\n VAL_LOSSES = \"val_losses.npy\"\n VAL_ERRORS = \"val_errors.npy\"\n\n LEG_MEAN = \"mean.npy\"\n LEG_STD = \"std.npy\"\n IM_MEAN = \"im_mean.npy\"\n IM_STD = \"im_std.npy\"\n IM_DEPTH_SUB_MEAN = \"im_depth_sub_mean.npy\"\n IM_DEPTH_SUB_STD = \"im_depth_sub_std.npy\"\n POSE_MEAN = \"pose_mean.npy\"\n POSE_STD = \"pose_std.npy\"\n\n FINAL_MODEL = \"model.ckpt\"\n INTER_MODEL = \"model_{}.ckpt\"\n\n SAVED_ARCH = \"architecture.json\"\n SAVED_CFG = \"config.json\"\n" }, { "path": "gqcnn/utils/policy_exceptions.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nExceptions that can be thrown by sub-classes of `GraspingPolicy`.\n\nAuthor\n------\nVishal Satish\n\"\"\"\n\n\nclass NoValidGraspsException(Exception):\n \"\"\"Exception for when antipodal point pairs can be found in the depth\n image but none are valid grasps that can be executed.\"\"\"\n\n def __init__(self,\n in_collision=True,\n not_confident=False,\n *args,\n **kwargs):\n self.in_collision = in_collision\n self.not_confident = not_confident\n Exception.__init__(self, *args, **kwargs)\n\n\nclass NoAntipodalPairsFoundException(Exception):\n \"\"\"Exception for when no antipodal point pairs can be found in the depth\n image.\"\"\"\n pass\n" }, { "path": "gqcnn/utils/train_stats_logger.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nHandles logging of various training statistics.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nimport os\n\nimport numpy as np\n\nfrom .enums import GQCNNFilenames\n\n\nclass TrainStatsLogger(object):\n \"\"\"Logger for training statistics.\"\"\"\n\n def __init__(self, experiment_dir):\n \"\"\"\n Parameters\n ----------\n experiment_dir : str\n The experiment directory to save statistics to.\n \"\"\"\n self.experiment_dir = experiment_dir\n self.train_eval_iters = []\n self.train_losses = []\n self.train_errors = []\n self.total_train_errors = []\n self.total_train_losses = []\n self.val_eval_iters = []\n self.val_losses = []\n self.val_errors = []\n self.val_losses = []\n self.learning_rates = []\n\n def log(self):\n \"\"\"Flush all of the statistics to the given experiment directory.\"\"\"\n np.save(os.path.join(self.experiment_dir, GQCNNFilenames.TRAIN_ITERS),\n self.train_eval_iters)\n np.save(os.path.join(self.experiment_dir, GQCNNFilenames.TRAIN_LOSSES),\n self.train_losses)\n np.save(os.path.join(self.experiment_dir, GQCNNFilenames.TRAIN_ERRORS),\n self.train_errors)\n np.save(\n os.path.join(self.experiment_dir,\n GQCNNFilenames.TOTAL_TRAIN_ERRORS),\n self.total_train_errors)\n np.save(\n os.path.join(self.experiment_dir,\n GQCNNFilenames.TOTAL_TRAIN_LOSSES),\n self.total_train_losses)\n np.save(os.path.join(self.experiment_dir, GQCNNFilenames.VAL_ITERS),\n self.val_eval_iters)\n np.save(os.path.join(self.experiment_dir, GQCNNFilenames.VAL_LOSSES),\n self.val_losses)\n np.save(os.path.join(self.experiment_dir, GQCNNFilenames.VAL_ERRORS),\n self.val_errors)\n np.save(\n os.path.join(self.experiment_dir, GQCNNFilenames.LEARNING_RATES),\n self.learning_rates)\n\n def update(self, **stats):\n \"\"\"Update training statistics.\n\n Note\n ----\n Any statistic that is `None` in the argument dict will not be updated.\n\n Parameters\n ----------\n stats : dict\n Dict of statistics to be updated.\n \"\"\"\n for stat, val in stats.items():\n if stat == \"train_eval_iter\":\n if val is not None:\n self.train_eval_iters.append(val)\n elif stat == \"train_loss\":\n if val is not None:\n self.train_losses.append(val)\n elif stat == \"train_error\":\n if val is not None:\n self.train_errors.append(val)\n elif stat == \"total_train_error\":\n if val is not None:\n self.total_train_errors.append(val)\n elif stat == \"total_train_loss\":\n if val is not None:\n self.total_train_losses.append(val)\n elif stat == \"val_eval_iter\":\n if val is not None:\n self.val_eval_iters.append(val)\n elif stat == \"val_loss\":\n if val is not None:\n self.val_losses.append(val)\n elif stat == \"val_error\":\n if val is not None:\n self.val_errors.append(val)\n elif stat == \"learning_rate\":\n if val is not None:\n self.learning_rates.append(val)\n" }, { "path": "gqcnn/utils/utils.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nSimple utility functions.\n\nAuthors\n-------\nJeff Mahler, Vishal Satish, Lucas Manuelli\n\"\"\"\nfrom functools import reduce\nimport os\nimport sys\n\nimport numpy as np\nimport skimage.transform as skt\n\nfrom autolab_core import Logger\nfrom .enums import GripperMode\n\n# Set up logger.\nlogger = Logger.get_logger(\"gqcnn/utils/utils.py\")\n\n\ndef is_py2():\n return sys.version[0] == \"2\"\n\n\ndef set_cuda_visible_devices(gpu_list):\n \"\"\"Sets CUDA_VISIBLE_DEVICES environment variable to only show certain\n gpus.\n\n Note\n ----\n If gpu_list is empty does nothing.\n\n Parameters\n ----------\n gpu_list : list\n List of gpus to set as visible.\n \"\"\"\n if len(gpu_list) == 0:\n return\n\n cuda_visible_devices = \"\"\n for gpu in gpu_list:\n cuda_visible_devices += str(gpu) + \",\"\n\n logger.info(\n \"Setting CUDA_VISIBLE_DEVICES = {}\".format(cuda_visible_devices))\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = cuda_visible_devices\n\n\ndef pose_dim(gripper_mode):\n \"\"\"Returns the dimensions of the pose vector for the given\n gripper mode.\n\n Parameters\n ----------\n gripper_mode: :obj:`GripperMode`\n Enum for gripper mode, see optimizer_constants.py for all possible\n gripper modes.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Sliced pose_data corresponding to gripper mode.\n \"\"\"\n if gripper_mode == GripperMode.PARALLEL_JAW:\n return 1\n elif gripper_mode == GripperMode.SUCTION:\n return 2\n elif gripper_mode == GripperMode.MULTI_SUCTION:\n return 1\n elif gripper_mode == GripperMode.LEGACY_PARALLEL_JAW:\n return 1\n elif gripper_mode == GripperMode.LEGACY_SUCTION:\n return 2\n else:\n raise ValueError(\n \"Gripper mode '{}' not supported.\".format(gripper_mode))\n\n\ndef read_pose_data(pose_arr, gripper_mode):\n \"\"\"Read the pose data and slice it according to the specified gripper mode.\n\n Parameters\n ----------\n pose_arr: :obj:`numpy.ndarray`\n Full pose data array read in from file.\n gripper_mode: :obj:`GripperMode`\n Enum for gripper mode, see optimizer_constants.py for all possible\n gripper modes.\n\n Returns\n -------\n :obj:`numpy.ndarray`\n Sliced pose_data corresponding to input data mode.\n \"\"\"\n if gripper_mode == GripperMode.PARALLEL_JAW:\n if pose_arr.ndim == 1:\n return pose_arr[2:3]\n else:\n return pose_arr[:, 2:3]\n elif gripper_mode == GripperMode.SUCTION:\n if pose_arr.ndim == 1:\n return np.r_[pose_arr[2], pose_arr[4]]\n else:\n return np.c_[pose_arr[:, 2], pose_arr[:, 4]]\n elif gripper_mode == GripperMode.MULTI_SUCTION:\n if pose_arr.ndim == 1:\n return pose_arr[2:3]\n else:\n return pose_arr[:, 2:3]\n elif gripper_mode == GripperMode.LEGACY_PARALLEL_JAW:\n if pose_arr.ndim == 1:\n return pose_arr[2:3]\n else:\n return pose_arr[:, 2:3]\n elif gripper_mode == GripperMode.LEGACY_SUCTION:\n if pose_arr.ndim == 1:\n return pose_arr[2:4]\n else:\n return pose_arr[:, 2:4]\n else:\n raise ValueError(\n \"Gripper mode '{}' not supported.\".format(gripper_mode))\n\n\ndef reduce_shape(shape):\n \"\"\"Get shape of a layer for flattening.\"\"\"\n shape = [x.value for x in shape[1:]]\n f = lambda x, y: 1 if y is None else x * y # noqa: E731\n return reduce(f, shape, 1)\n\n\ndef weight_name_to_layer_name(weight_name):\n \"\"\"Convert the name of weights to the layer name.\"\"\"\n tokens = weight_name.split(\"_\")\n type_name = tokens[-1]\n\n # Modern naming convention.\n if type_name == \"weights\" or type_name == \"bias\":\n if len(tokens) >= 3 and tokens[-3] == \"input\":\n return weight_name[:weight_name.rfind(\"input\") - 1]\n return weight_name[:weight_name.rfind(type_name) - 1]\n # Legacy.\n if type_name == \"im\":\n return weight_name[:-4]\n if type_name == \"pose\":\n return weight_name[:-6]\n return weight_name[:-1]\n\n\ndef imresize(image, size, interp=\"nearest\"):\n \"\"\"Wrapper over `skimage.transform.resize` to mimic `scipy.misc.imresize`.\n Copied from https://github.com/BerkeleyAutomation/autolab_core/blob/master/autolab_core/image.py#L32. # noqa: E501\n\n Since `scipy.misc.imresize` has been removed in version 1.3.*, instead use\n `skimage.transform.resize`. The \"lanczos\" and \"cubic\" interpolation methods\n are not supported by `skimage.transform.resize`, however there is now\n \"biquadratic\", \"biquartic\", and \"biquintic\".\n\n Parameters\n ----------\n image : :obj:`numpy.ndarray`\n The image to resize.\n\n size : int, float, or tuple\n * int - Percentage of current size.\n * float - Fraction of current size.\n * tuple - Size of the output image.\n\n interp : :obj:`str`, optional\n Interpolation to use for re-sizing (\"neartest\", \"bilinear\",\n \"biquadratic\", \"bicubic\", \"biquartic\", \"biquintic\"). Default is\n \"nearest\".\n\n Returns\n -------\n :obj:`np.ndarray`\n The resized image.\n \"\"\"\n skt_interp_map = {\n \"nearest\": 0,\n \"bilinear\": 1,\n \"biquadratic\": 2,\n \"bicubic\": 3,\n \"biquartic\": 4,\n \"biquintic\": 5\n }\n if interp in (\"lanczos\", \"cubic\"):\n raise ValueError(\"'lanczos' and 'cubic'\"\n \" interpolation are no longer supported.\")\n assert interp in skt_interp_map, (\"Interpolation '{}' not\"\n \" supported.\".format(interp))\n\n if isinstance(size, (tuple, list)):\n output_shape = size\n elif isinstance(size, (float)):\n np_shape = np.asarray(image.shape).astype(np.float32)\n np_shape[0:2] *= size\n output_shape = tuple(np_shape.astype(int))\n elif isinstance(size, (int)):\n np_shape = np.asarray(image.shape).astype(np.float32)\n np_shape[0:2] *= size / 100.0\n output_shape = tuple(np_shape.astype(int))\n else:\n raise ValueError(\"Invalid type for size '{}'.\".format(type(size)))\n\n return skt.resize(image,\n output_shape,\n order=skt_interp_map[interp],\n anti_aliasing=False,\n mode=\"constant\")\n" }, { "path": "gqcnn/version.py", "content": "__version__ = \"1.3.0\"\n" }, { "path": "launch/grasp_planning_service.launch", "content": "\n\n\n \n \n \n \n \n \n \n \n \n \t\n \t\n \t\n \n\n" }, { "path": "msg/Action.msg", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nuint32 width\nuint32 height\nuint8[] mask_data\nstring action_type\nfloat32[] action_data\nfloat32 confidence\n" }, { "path": "msg/BoundingBox.msg", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nfloat64 minX\nfloat64 minY\nfloat64 maxX\nfloat64 maxY\n" }, { "path": "msg/GQCNNGrasp.msg", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\ngeometry_msgs/Pose pose\nfloat64 q_value\n\nuint8 PARALLEL_JAW=0\nuint8 SUCTION=1\nuint8 grasp_type\n\nfloat64[2] center_px\nfloat64 angle\nfloat64 depth\nsensor_msgs/Image thumbnail\n" }, { "path": "msg/Observation.msg", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nuint32 width\nuint32 height\nfloat32[] image_data\n" }, { "path": "package.xml", "content": "\n\n\n gqcnn\n 1.3.0\n ROS package for deploying Grasp Quality Convolutional Neural Networks (GQ-CNNs).\n\n \n \n \n Vishal Satish\n\n\n \n \n \n Regents\n\n\n \n \n \n \n\n\n \n \n \n \n\n\n \n \n \n \n \n \n \n \n \n \n \n catkin\n rospy\n rospy\n message_generation\n message_runtime\n\n \n \n \n\n \n\n" }, { "path": "post-checkout", "content": "#!/bin/sh\n#\n# An example hook script to prepare a packed repository for use over\n# dumb transports.\n#\n# To enable this hook, rename this file to \"post-update\".\n\nfind . -name \"*.pyc\" -exec rm '{}' ';'" }, { "path": "requirements/cpu_requirements.txt", "content": "autolab-core\nautolab-perception\nvisualization\nnumpy\nopencv-python\nscipy\nmatplotlib\ntensorflow<=1.15.0\nscikit-learn\nscikit-image\ngputil\npsutil\n" }, { "path": "requirements/docs_requirements.txt", "content": "sphinx\nsphinxcontrib-napoleon\nsphinx_rtd_theme\n\n" }, { "path": "requirements/gpu_requirements.txt", "content": "autolab-core\nautolab-perception\nvisualization\nnumpy\nopencv-python\nscipy\nmatplotlib\ntensorflow-gpu<=1.15.0\nscikit-learn\nscikit-image\ngputil\npsutil\n" }, { "path": "ros_nodes/grasp_planner_node.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nROS Server for planning GQ-CNN grasps.\n\nAuthor\n-----\nVishal Satish & Jeff Mahler\n\"\"\"\nimport json\nimport math\nimport os\nimport time\n\nfrom cv_bridge import CvBridge, CvBridgeError\nimport numpy as np\nimport rospy\n\nfrom autolab_core import (YamlConfig, CameraIntrinsics, ColorImage,\n DepthImage, BinaryImage, RgbdImage)\nfrom visualization import Visualizer2D as vis\nfrom gqcnn.grasping import (Grasp2D, SuctionPoint2D, RgbdImageState,\n RobustGraspingPolicy,\n CrossEntropyRobustGraspingPolicy,\n FullyConvolutionalGraspingPolicyParallelJaw,\n FullyConvolutionalGraspingPolicySuction)\nfrom gqcnn.utils import GripperMode, NoValidGraspsException\n\nfrom geometry_msgs.msg import PoseStamped\nfrom std_msgs.msg import Header\nfrom gqcnn.srv import (GQCNNGraspPlanner, GQCNNGraspPlannerBoundingBox,\n GQCNNGraspPlannerSegmask)\nfrom gqcnn.msg import GQCNNGrasp\n\n\nclass GraspPlanner(object):\n\n def __init__(self, cfg, cv_bridge, grasping_policy, grasp_pose_publisher):\n \"\"\"\n Parameters\n ----------\n cfg : dict\n Dictionary of configuration parameters.\n cv_bridge: :obj:`CvBridge`\n ROS `CvBridge`.\n grasping_policy: :obj:`GraspingPolicy`\n Grasping policy to use.\n grasp_pose_publisher: :obj:`Publisher`\n ROS publisher to publish pose of planned grasp for visualization.\n \"\"\"\n self.cfg = cfg\n self.cv_bridge = cv_bridge\n self.grasping_policy = grasping_policy\n self.grasp_pose_publisher = grasp_pose_publisher\n\n # Set minimum input dimensions.\n policy_type = \"cem\"\n if \"type\" in self.cfg[\"policy\"]:\n policy_type = self.cfg[\"policy\"][\"type\"]\n\n fully_conv_policy_types = {\"fully_conv_suction\", \"fully_conv_pj\"}\n if policy_type in fully_conv_policy_types:\n self.min_width = self.cfg[\"policy\"][\"gqcnn_recep_w\"]\n self.min_height = self.cfg[\"policy\"][\"gqcnn_recep_h\"]\n else:\n pad = max(\n math.ceil(\n np.sqrt(2) *\n (float(self.cfg[\"policy\"][\"metric\"][\"crop_width\"]) / 2)),\n math.ceil(\n np.sqrt(2) *\n (float(self.cfg[\"policy\"][\"metric\"][\"crop_height\"]) / 2)))\n self.min_width = 2 * pad + self.cfg[\"policy\"][\"metric\"][\n \"crop_width\"]\n self.min_height = 2 * pad + self.cfg[\"policy\"][\"metric\"][\n \"crop_height\"]\n\n def read_images(self, req):\n \"\"\"Reads images from a ROS service request.\n\n Parameters\n ---------\n req: :obj:`ROS ServiceRequest`\n ROS ServiceRequest for grasp planner service.\n \"\"\"\n # Get the raw depth and color images as ROS `Image` objects.\n raw_color = req.color_image\n raw_depth = req.depth_image\n\n # Get the raw camera info as ROS `CameraInfo`.\n raw_camera_info = req.camera_info\n\n # Wrap the camera info in a BerkeleyAutomation/autolab_core\n # `CameraIntrinsics` object.\n camera_intr = CameraIntrinsics(\n raw_camera_info.header.frame_id, raw_camera_info.K[0],\n raw_camera_info.K[4], raw_camera_info.K[2], raw_camera_info.K[5],\n raw_camera_info.K[1], raw_camera_info.height,\n raw_camera_info.width)\n\n # Create wrapped BerkeleyAutomation/autolab_core RGB and depth images\n # by unpacking the ROS images using ROS `CvBridge`\n try:\n color_im = ColorImage(self.cv_bridge.imgmsg_to_cv2(\n raw_color, \"rgb8\"),\n frame=camera_intr.frame)\n depth_im = DepthImage(self.cv_bridge.imgmsg_to_cv2(\n raw_depth, desired_encoding=\"passthrough\"),\n frame=camera_intr.frame)\n except CvBridgeError as cv_bridge_exception:\n rospy.logerr(cv_bridge_exception)\n\n # Check image sizes.\n if color_im.height != depth_im.height or \\\n color_im.width != depth_im.width:\n msg = (\"Color image and depth image must be the same shape! Color\"\n \" is %d x %d but depth is %d x %d\") % (\n color_im.height, color_im.width, depth_im.height,\n depth_im.width)\n rospy.logerr(msg)\n raise rospy.ServiceException(msg)\n\n if (color_im.height < self.min_height\n or color_im.width < self.min_width):\n msg = (\"Color image is too small! Must be at least %d x %d\"\n \" resolution but the requested image is only %d x %d\") % (\n self.min_height, self.min_width, color_im.height,\n color_im.width)\n rospy.logerr(msg)\n raise rospy.ServiceException(msg)\n\n return color_im, depth_im, camera_intr\n\n def plan_grasp(self, req):\n \"\"\"Grasp planner request handler.\n\n Parameters\n ---------\n req: :obj:`ROS ServiceRequest`\n ROS `ServiceRequest` for grasp planner service.\n \"\"\"\n color_im, depth_im, camera_intr = self.read_images(req)\n return self._plan_grasp(color_im, depth_im, camera_intr)\n\n def plan_grasp_bb(self, req):\n \"\"\"Grasp planner request handler.\n\n Parameters\n ---------\n req: :obj:`ROS ServiceRequest`\n `ROS ServiceRequest` for grasp planner service.\n \"\"\"\n color_im, depth_im, camera_intr = self.read_images(req)\n return self._plan_grasp(color_im,\n depth_im,\n camera_intr,\n bounding_box=req.bounding_box)\n\n def plan_grasp_segmask(self, req):\n \"\"\"Grasp planner request handler.\n\n Parameters\n ---------\n req: :obj:`ROS ServiceRequest`\n ROS `ServiceRequest` for grasp planner service.\n \"\"\"\n color_im, depth_im, camera_intr = self.read_images(req)\n raw_segmask = req.segmask\n try:\n segmask = BinaryImage(self.cv_bridge.imgmsg_to_cv2(\n raw_segmask, desired_encoding=\"passthrough\"),\n frame=camera_intr.frame)\n except CvBridgeError as cv_bridge_exception:\n rospy.logerr(cv_bridge_exception)\n if color_im.height != segmask.height or \\\n color_im.width != segmask.width:\n msg = (\"Images and segmask must be the same shape! Color image is\"\n \" %d x %d but segmask is %d x %d\") % (\n color_im.height, color_im.width, segmask.height,\n segmask.width)\n rospy.logerr(msg)\n raise rospy.ServiceException(msg)\n\n return self._plan_grasp(color_im,\n depth_im,\n camera_intr,\n segmask=segmask)\n\n def _plan_grasp(self,\n color_im,\n depth_im,\n camera_intr,\n bounding_box=None,\n segmask=None):\n \"\"\"Grasp planner request handler.\n\n Parameters\n ---------\n req: :obj:`ROS ServiceRequest`\n ROS `ServiceRequest` for grasp planner service.\n \"\"\"\n rospy.loginfo(\"Planning Grasp\")\n\n # Inpaint images.\n color_im = color_im.inpaint(\n rescale_factor=self.cfg[\"inpaint_rescale_factor\"])\n depth_im = depth_im.inpaint(\n rescale_factor=self.cfg[\"inpaint_rescale_factor\"])\n\n # Init segmask.\n if segmask is None:\n segmask = BinaryImage(255 *\n np.ones(depth_im.shape).astype(np.uint8),\n frame=color_im.frame)\n\n # Visualize.\n if self.cfg[\"vis\"][\"color_image\"]:\n vis.imshow(color_im)\n vis.show()\n if self.cfg[\"vis\"][\"depth_image\"]:\n vis.imshow(depth_im)\n vis.show()\n if self.cfg[\"vis\"][\"segmask\"] and segmask is not None:\n vis.imshow(segmask)\n vis.show()\n\n # Aggregate color and depth images into a single\n # BerkeleyAutomation/autolab_core `RgbdImage`.\n rgbd_im = RgbdImage.from_color_and_depth(color_im, depth_im)\n\n # Mask bounding box.\n if bounding_box is not None:\n # Calc bb parameters.\n min_x = bounding_box.minX\n min_y = bounding_box.minY\n max_x = bounding_box.maxX\n max_y = bounding_box.maxY\n\n # Contain box to image->don't let it exceed image height/width\n # bounds.\n if min_x < 0:\n min_x = 0\n if min_y < 0:\n min_y = 0\n if max_x > rgbd_im.width:\n max_x = rgbd_im.width\n if max_y > rgbd_im.height:\n max_y = rgbd_im.height\n\n # Mask.\n bb_segmask_arr = np.zeros([rgbd_im.height, rgbd_im.width])\n bb_segmask_arr[min_y:max_y, min_x:max_x] = 255\n bb_segmask = BinaryImage(bb_segmask_arr.astype(np.uint8),\n segmask.frame)\n segmask = segmask.mask_binary(bb_segmask)\n\n # Visualize.\n if self.cfg[\"vis\"][\"rgbd_state\"]:\n masked_rgbd_im = rgbd_im.mask_binary(segmask)\n vis.figure()\n vis.subplot(1, 2, 1)\n vis.imshow(masked_rgbd_im.color)\n vis.subplot(1, 2, 2)\n vis.imshow(masked_rgbd_im.depth)\n vis.show()\n\n # Create an `RgbdImageState` with the cropped `RgbdImage` and\n # `CameraIntrinsics`.\n rgbd_state = RgbdImageState(rgbd_im, camera_intr, segmask=segmask)\n\n # Execute policy.\n try:\n return self.execute_policy(rgbd_state, self.grasping_policy,\n self.grasp_pose_publisher,\n camera_intr.frame)\n except NoValidGraspsException:\n rospy.logerr(\n (\"While executing policy found no valid grasps from sampled\"\n \" antipodal point pairs. Aborting Policy!\"))\n raise rospy.ServiceException(\n (\"While executing policy found no valid grasps from sampled\"\n \" antipodal point pairs. Aborting Policy!\"))\n\n def execute_policy(self, rgbd_image_state, grasping_policy,\n grasp_pose_publisher, pose_frame):\n \"\"\"Executes a grasping policy on an `RgbdImageState`.\n\n Parameters\n ----------\n rgbd_image_state: :obj:`RgbdImageState`\n `RgbdImageState` to encapsulate\n depth and color image along with camera intrinsics.\n grasping_policy: :obj:`GraspingPolicy`\n Grasping policy to use.\n grasp_pose_publisher: :obj:`Publisher`\n ROS publisher to publish pose of planned grasp for visualization.\n pose_frame: :obj:`str`\n Frame of reference to publish pose in.\n \"\"\"\n # Execute the policy\"s action.\n grasp_planning_start_time = time.time()\n grasp = grasping_policy(rgbd_image_state)\n\n # Create `GQCNNGrasp` return msg and populate it.\n gqcnn_grasp = GQCNNGrasp()\n gqcnn_grasp.q_value = grasp.q_value\n gqcnn_grasp.pose = grasp.grasp.pose().pose_msg\n if isinstance(grasp.grasp, Grasp2D):\n gqcnn_grasp.grasp_type = GQCNNGrasp.PARALLEL_JAW\n elif isinstance(grasp.grasp, SuctionPoint2D):\n gqcnn_grasp.grasp_type = GQCNNGrasp.SUCTION\n else:\n rospy.logerr(\"Grasp type not supported!\")\n raise rospy.ServiceException(\"Grasp type not supported!\")\n\n # Store grasp representation in image space.\n gqcnn_grasp.center_px[0] = grasp.grasp.center[0]\n gqcnn_grasp.center_px[1] = grasp.grasp.center[1]\n gqcnn_grasp.angle = grasp.grasp.angle\n gqcnn_grasp.depth = grasp.grasp.depth\n gqcnn_grasp.thumbnail = grasp.image.rosmsg\n\n # Create and publish the pose alone for easy visualization of grasp\n # pose in Rviz.\n pose_stamped = PoseStamped()\n pose_stamped.pose = grasp.grasp.pose().pose_msg\n header = Header()\n header.stamp = rospy.Time.now()\n header.frame_id = pose_frame\n pose_stamped.header = header\n grasp_pose_publisher.publish(pose_stamped)\n\n # Return `GQCNNGrasp` msg.\n rospy.loginfo(\"Total grasp planning time: \" +\n str(time.time() - grasp_planning_start_time) + \" secs.\")\n\n return gqcnn_grasp\n\n\nif __name__ == \"__main__\":\n # Initialize the ROS node.\n rospy.init_node(\"Grasp_Sampler_Server\")\n\n # Initialize `CvBridge`.\n cv_bridge = CvBridge()\n\n # Get configs.\n model_name = rospy.get_param(\"~model_name\")\n model_dir = rospy.get_param(\"~model_dir\")\n fully_conv = rospy.get_param(\"~fully_conv\")\n if model_dir.lower() == \"default\":\n model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"../models\")\n model_dir = os.path.join(model_dir, model_name)\n model_config = json.load(open(os.path.join(model_dir, \"config.json\"), \"r\"))\n try:\n gqcnn_config = model_config[\"gqcnn\"]\n gripper_mode = gqcnn_config[\"gripper_mode\"]\n except KeyError:\n gqcnn_config = model_config[\"gqcnn_config\"]\n input_data_mode = gqcnn_config[\"input_data_mode\"]\n if input_data_mode == \"tf_image\":\n gripper_mode = GripperMode.LEGACY_PARALLEL_JAW\n elif input_data_mode == \"tf_image_suction\":\n gripper_mode = GripperMode.LEGACY_SUCTION\n elif input_data_mode == \"suction\":\n gripper_mode = GripperMode.SUCTION\n elif input_data_mode == \"multi_suction\":\n gripper_mode = GripperMode.MULTI_SUCTION\n elif input_data_mode == \"parallel_jaw\":\n gripper_mode = GripperMode.PARALLEL_JAW\n else:\n raise ValueError(\n \"Input data mode {} not supported!\".format(input_data_mode))\n\n # Set config.\n if fully_conv:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/ros/fc_gqcnn_suction.yaml\")\n else:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/ros/gqcnn_suction.yaml\")\n if (gripper_mode == GripperMode.LEGACY_PARALLEL_JAW\n or gripper_mode == GripperMode.PARALLEL_JAW):\n if fully_conv:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/ros/fc_gqcnn_pj.yaml\")\n else:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/examples/ros/gqcnn_pj.yaml\")\n\n # Read config.\n cfg = YamlConfig(config_filename)\n policy_cfg = cfg[\"policy\"]\n policy_cfg[\"metric\"][\"gqcnn_model\"] = model_dir\n\n # Create publisher to publish pose of final grasp.\n grasp_pose_publisher = rospy.Publisher(\"/gqcnn_grasp/pose\",\n PoseStamped,\n queue_size=10)\n\n # Create a grasping policy.\n rospy.loginfo(\"Creating Grasping Policy\")\n if fully_conv:\n # TODO(vsatish): We should really be doing this in some factory policy.\n if policy_cfg[\"type\"] == \"fully_conv_suction\":\n grasping_policy = \\\n FullyConvolutionalGraspingPolicySuction(policy_cfg)\n elif policy_cfg[\"type\"] == \"fully_conv_pj\":\n grasping_policy = \\\n FullyConvolutionalGraspingPolicyParallelJaw(policy_cfg)\n else:\n raise ValueError(\n \"Invalid fully-convolutional policy type: {}\".format(\n policy_cfg[\"type\"]))\n else:\n policy_type = \"cem\"\n if \"type\" in policy_cfg:\n policy_type = policy_cfg[\"type\"]\n if policy_type == \"ranking\":\n grasping_policy = RobustGraspingPolicy(policy_cfg)\n elif policy_type == \"cem\":\n grasping_policy = CrossEntropyRobustGraspingPolicy(policy_cfg)\n else:\n raise ValueError(\"Invalid policy type: {}\".format(policy_type))\n\n # Create a grasp planner.\n grasp_planner = GraspPlanner(cfg, cv_bridge, grasping_policy,\n grasp_pose_publisher)\n\n # Initialize the ROS service.\n grasp_planning_service = rospy.Service(\"grasp_planner\", GQCNNGraspPlanner,\n grasp_planner.plan_grasp)\n grasp_planning_service_bb = rospy.Service(\"grasp_planner_bounding_box\",\n GQCNNGraspPlannerBoundingBox,\n grasp_planner.plan_grasp_bb)\n grasp_planning_service_segmask = rospy.Service(\n \"grasp_planner_segmask\", GQCNNGraspPlannerSegmask,\n grasp_planner.plan_grasp_segmask)\n rospy.loginfo(\"Grasping Policy Initialized\")\n\n # Spin forever.\n rospy.spin()\n" }, { "path": "scripts/docker/build-docker.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n# Build the CPU and GPU docker images.\n\ngit archive --format=tar -o docker/gqcnn.tar --prefix=gqcnn/ master\ndocker build --no-cache -t gqcnn/gpu -f docker/gpu/Dockerfile .\ndocker build --no-cache -t gqcnn/cpu -f docker/cpu/Dockerfile .\nrm docker/gqcnn.tar\n" }, { "path": "scripts/policies/run_all_dex-net_2.0_examples.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nset -e\n\necho \"RUNNING EXAMPLE 1\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_0.npy --segmask data/examples/single_object/primesense/segmask_0.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 2\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_1.npy --segmask data/examples/single_object/primesense/segmask_1.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 3\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_2.npy --segmask data/examples/single_object/primesense/segmask_2.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 4\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_3.npy --segmask data/examples/single_object/primesense/segmask_3.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 5\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_4.npy --segmask data/examples/single_object/primesense/segmask_4.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 6\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_5.npy --segmask data/examples/single_object/primesense/segmask_5.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 7\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_6.npy --segmask data/examples/single_object/primesense/segmask_6.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 8\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_7.npy --segmask data/examples/single_object/primesense/segmask_7.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 9\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_8.npy --segmask data/examples/single_object/primesense/segmask_8.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\necho \"RUNNING EXAMPLE 10\"\npython examples/policy.py GQCNN-2.0 --depth_image data/examples/single_object/primesense/depth_9.npy --segmask data/examples/single_object/primesense/segmask_9.png --config_filename cfg/examples/replication/dex-net_2.0.yaml\n\n" }, { "path": "scripts/policies/run_all_dex-net_2.1_examples.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nset -e\n\necho \"RUNNING EXAMPLE 1\"\npython examples/policy.py GQCNN-2.1 --depth_image data/examples/clutter/primesense/depth_0.npy --segmask data/examples/clutter/primesense/segmask_0.png --config_filename cfg/examples/replication/dex-net_2.1.yaml\n\necho \"RUNNING EXAMPLE 2\"\npython examples/policy.py GQCNN-2.1 --depth_image data/examples/clutter/primesense/depth_1.npy --segmask data/examples/clutter/primesense/segmask_1.png --config_filename cfg/examples/replication/dex-net_2.1.yaml\n\necho \"RUNNING EXAMPLE 3\"\npython examples/policy.py GQCNN-2.1 --depth_image data/examples/clutter/primesense/depth_2.npy --segmask data/examples/clutter/primesense/segmask_2.png --config_filename cfg/examples/replication/dex-net_2.1.yaml\n\necho \"RUNNING EXAMPLE 4\"\npython examples/policy.py GQCNN-2.1 --depth_image data/examples/clutter/primesense/depth_3.npy --segmask data/examples/clutter/primesense/segmask_3.png --config_filename cfg/examples/replication/dex-net_2.1.yaml\n\necho \"RUNNING EXAMPLE 5\"\npython examples/policy.py GQCNN-2.1 --depth_image data/examples/clutter/primesense/depth_4.npy --segmask data/examples/clutter/primesense/segmask_4.png --config_filename cfg/examples/replication/dex-net_2.1.yaml\n\n" }, { "path": "scripts/policies/run_all_dex-net_3.0_examples.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nset -e\n\necho \"RUNNING EXAMPLE 1\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_0.npy --segmask data/examples/single_object/primesense/segmask_0.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 2\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_1.npy --segmask data/examples/single_object/primesense/segmask_1.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 3\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_2.npy --segmask data/examples/single_object/primesense/segmask_2.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 4\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_3.npy --segmask data/examples/single_object/primesense/segmask_3.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 5\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_4.npy --segmask data/examples/single_object/primesense/segmask_4.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 6\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_5.npy --segmask data/examples/single_object/primesense/segmask_5.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 7\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_6.npy --segmask data/examples/single_object/primesense/segmask_6.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 8\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_7.npy --segmask data/examples/single_object/primesense/segmask_7.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 9\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_8.npy --segmask data/examples/single_object/primesense/segmask_8.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\necho \"RUNNING EXAMPLE 10\"\npython examples/policy.py GQCNN-3.0 --depth_image data/examples/single_object/primesense/depth_9.npy --segmask data/examples/single_object/primesense/segmask_9.png --config_filename cfg/examples/replication/dex-net_3.0.yaml\n\n" }, { "path": "scripts/policies/run_all_dex-net_4.0_fc_pj_examples.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nset -e\n\necho \"RUNNING EXAMPLE 1\"\npython examples/policy.py FC-GQCNN-4.0-PJ --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_0.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_0.png --config_filename cfg/examples/replication/dex-net_4.0_fc_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 2\"\npython examples/policy.py FC-GQCNN-4.0-PJ --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_1.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_1.png --config_filename cfg/examples/replication/dex-net_4.0_fc_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 3\"\npython examples/policy.py FC-GQCNN-4.0-PJ --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_2.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_2.png --config_filename cfg/examples/replication/dex-net_4.0_fc_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 4\"\npython examples/policy.py FC-GQCNN-4.0-PJ --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_3.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_3.png --config_filename cfg/examples/replication/dex-net_4.0_fc_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 5\"\npython examples/policy.py FC-GQCNN-4.0-PJ --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_4.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_4.png --config_filename cfg/examples/replication/dex-net_4.0_fc_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n" }, { "path": "scripts/policies/run_all_dex-net_4.0_fc_suction_examples.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nset -e\n\necho \"RUNNING EXAMPLE 1\"\npython examples/policy.py FC-GQCNN-4.0-SUCTION --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_0.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_0.png --config_filename cfg/examples/replication/dex-net_4.0_fc_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 2\"\npython examples/policy.py FC-GQCNN-4.0-SUCTION --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_1.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_1.png --config_filename cfg/examples/replication/dex-net_4.0_fc_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 3\"\npython examples/policy.py FC-GQCNN-4.0-SUCTION --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_2.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_2.png --config_filename cfg/examples/replication/dex-net_4.0_fc_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 4\"\npython examples/policy.py FC-GQCNN-4.0-SUCTION --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_3.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_3.png --config_filename cfg/examples/replication/dex-net_4.0_fc_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 5\"\npython examples/policy.py FC-GQCNN-4.0-SUCTION --fully_conv --depth_image data/examples/clutter/phoxi/fcgqcnn/depth_4.npy --segmask data/examples/clutter/phoxi/fcgqcnn/segmask_4.png --config_filename cfg/examples/replication/dex-net_4.0_fc_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n" }, { "path": "scripts/policies/run_all_dex-net_4.0_pj_examples.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nset -e\n\necho \"RUNNING EXAMPLE 1\"\npython examples/policy.py GQCNN-4.0-PJ --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_0.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_0.png --config_filename cfg/examples/replication/dex-net_4.0_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 2\"\npython examples/policy.py GQCNN-4.0-PJ --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_1.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_1.png --config_filename cfg/examples/replication/dex-net_4.0_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 3\"\npython examples/policy.py GQCNN-4.0-PJ --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_2.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_2.png --config_filename cfg/examples/replication/dex-net_4.0_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 4\"\npython examples/policy.py GQCNN-4.0-PJ --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_3.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_3.png --config_filename cfg/examples/replication/dex-net_4.0_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 5\"\npython examples/policy.py GQCNN-4.0-PJ --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_4.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_4.png --config_filename cfg/examples/replication/dex-net_4.0_pj.yaml --camera_intr data/calib/phoxi/phoxi.intr\n" }, { "path": "scripts/policies/run_all_dex-net_4.0_suction_examples.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nset -e\n\necho \"RUNNING EXAMPLE 1\"\npython examples/policy.py GQCNN-4.0-SUCTION --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_0.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_0.png --config_filename cfg/examples/replication/dex-net_4.0_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 2\"\npython examples/policy.py GQCNN-4.0-SUCTION --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_1.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_1.png --config_filename cfg/examples/replication/dex-net_4.0_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 3\"\npython examples/policy.py GQCNN-4.0-SUCTION --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_2.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_2.png --config_filename cfg/examples/replication/dex-net_4.0_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 4\"\npython examples/policy.py GQCNN-4.0-SUCTION --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_3.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_3.png --config_filename cfg/examples/replication/dex-net_4.0_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n\necho \"RUNNING EXAMPLE 5\"\npython examples/policy.py GQCNN-4.0-SUCTION --depth_image data/examples/clutter/phoxi/dex-net_4.0/depth_4.npy --segmask data/examples/clutter/phoxi/dex-net_4.0/segmask_4.png --config_filename cfg/examples/replication/dex-net_4.0_suction.yaml --camera_intr data/calib/phoxi/phoxi.intr\n" }, { "path": "scripts/training/train_dex-net_2.0.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\npython tools/train.py data/training/dex-net_2.0 --config_filename cfg/train_dex-net_2.0.yaml --name GQCNN-2.0\n" }, { "path": "scripts/training/train_dex-net_2.1.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\necho \"Please contact Vishal Satish (vsatish@berkeley.edu) or Jeffrey Mahler (jmahler@berkeley.edu) for instructions on training Dex-Net 2.1.\" \n" }, { "path": "scripts/training/train_dex-net_3.0.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\npython tools/train.py data/training/dex-net_3.0 --config_filename cfg/train_dex-net_3.0.yaml --name GQCNN-3.0\n" }, { "path": "scripts/training/train_dex-net_4.0_fc_pj.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\npython tools/train.py data/training/dex-net_4.0_fc_pj --config_filename cfg/train_dex-net_4.0_fc_pj.yaml --name GQCNN-4.0-FC-PJ\n" }, { "path": "scripts/training/train_dex-net_4.0_fc_suction.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\npython tools/train.py data/training/dex-net_4.0_fc_suction --config_filename cfg/train_dex-net_4.0_fc_suction.yaml --name GQCNN-4.0-FC-Suction\n" }, { "path": "scripts/training/train_dex-net_4.0_pj.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\npython tools/train.py data/training/dexnet_4_pj --config_filename cfg/train_dex-net_4.0_pj.yaml --name GQCNN-4.0-PJ\n" }, { "path": "scripts/training/train_dex-net_4.0_suction.sh", "content": "#!/bin/bash\n\n# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\npython tools/train.py data/training/dexnet_4_suction --config_filename cfg/train_dex-net_4.0_suction.yaml --name GQCNN-4.0-Suction\n" }, { "path": "setup.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nSetup of `gqcnn` Python codebase.\n\nAuthor\n------\nVishal Satish & Jeff Mahler\n\"\"\"\nimport logging\nimport os\nfrom setuptools import setup, find_packages\nfrom setuptools.command.develop import develop\nfrom setuptools.command.install import install\nimport subprocess\nimport sys\n\nTF_MAX_VERSION = \"1.15.0\"\n\n# Set up logger.\nlogging.basicConfig() # Configure the root logger.\nlogger = logging.getLogger(\"setup.py\")\nlogger.setLevel(logging.INFO)\n\n\ndef get_tf_dep():\n # Check whether or not the Nvidia driver and GPUs are available and add the\n # corresponding Tensorflow dependency.\n tf_dep = \"tensorflow<={}\".format(TF_MAX_VERSION)\n try:\n gpus = subprocess.check_output(\n [\"nvidia-smi\", \"--query-gpu=gpu_name\",\n \"--format=csv\"]).decode().strip().split(\"\\n\")[1:]\n if len(gpus) > 0:\n tf_dep = \"tensorflow-gpu<={}\".format(TF_MAX_VERSION)\n else:\n no_device_msg = (\"Found Nvidia device driver but no\"\n \" devices...installing Tensorflow for CPU.\")\n logger.warning(no_device_msg)\n except OSError:\n no_driver_msg = (\"Could not find Nvidia device driver...installing\"\n \" Tensorflow for CPU.\")\n logger.warning(no_driver_msg)\n return tf_dep\n\n\n# TODO(vsatish): Use inheritance here.\nclass DevelopCmd(develop):\n user_options_custom = [\n (\"docker\", None, \"installing in Docker\"),\n ]\n user_options = getattr(develop, \"user_options\", []) + user_options_custom\n\n def initialize_options(self):\n develop.initialize_options(self)\n\n # Initialize options.\n self.docker = False\n\n def finalize_options(self):\n develop.finalize_options(self)\n\n def run(self):\n # Install Tensorflow dependency.\n if not self.docker:\n tf_dep = get_tf_dep()\n subprocess.Popen([sys.executable, \"-m\", \"pip\", \"install\",\n tf_dep]).wait()\n else:\n # If we're using Docker, this will already have been installed\n # explicitly through the correct `{cpu/gpu}_requirements.txt`;\n # there is no way to check for CUDA/GPUs at Docker build time\n # because there is no easy way to set the Nvidia runtime.\n # TODO(vsatish): Figure out why this isn't printed.\n skip_tf_msg = (\"Omitting Tensorflow dependency because of Docker\"\n \" installation.\")\n logger.warning(skip_tf_msg)\n\n # Run installation.\n develop.run(self)\n\n\nclass InstallCmd(install, object):\n user_options_custom = [\n (\"docker\", None, \"installing in Docker\"),\n ]\n user_options = getattr(install, \"user_options\", []) + user_options_custom\n\n def initialize_options(self):\n install.initialize_options(self)\n\n # Initialize options.\n self.docker = False\n\n def finalize_options(self):\n install.finalize_options(self)\n\n def run(self):\n # Install Tensorflow dependency.\n if not self.docker:\n tf_dep = get_tf_dep()\n subprocess.Popen([sys.executable, \"-m\", \"pip\", \"install\",\n tf_dep]).wait()\n else:\n # If we're using Docker, this will already have been installed\n # explicitly through the correct `{cpu/gpu}_requirements.txt`;\n # there is no way to check for CUDA/GPUs at Docker build time\n # because there is no easy way to set the Nvidia runtime.\n # TODO (vsatish): Figure out why this isn't printed.\n skip_tf_msg = (\"Omitting Tensorflow dependency because of Docker\"\n \" installation.\")\n logger.warning(skip_tf_msg)\n\n # Run installation.\n install.run(self)\n\n\nrequirements = [\n \"autolab-core\", \"autolab-perception\", \"visualization\", \"numpy\", \"scipy\",\n \"matplotlib\", \"opencv-python\", \"scikit-learn\", \"scikit-image\", \"psutil\",\n \"gputil\"\n]\n\nexec(\n open(\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"gqcnn/version.py\")).read())\n\nsetup(\n name=\"gqcnn\",\n version=__version__, # noqa F821\n description=(\"Project code for running Grasp Quality Convolutional\"\n \" Neural Networks\"),\n author=\"Vishal Satish\",\n author_email=\"vsatish@berkeley.edu\",\n license=\"Berkeley Copyright\",\n url=\"https://github.com/BerkeleyAutomation/gqcnn\",\n keywords=\"robotics grasping vision deep learning\",\n classifiers=[\n \"Development Status :: 4 - Beta\",\n \"Programming Language :: Python :: 3.5\",\n \"Programming Language :: Python :: 3.6\",\n \"Programming Language :: Python :: 3.7\",\n \"Natural Language :: English\", # yapf: disable\n \"Topic :: Scientific/Engineering\"\n ],\n packages=find_packages(),\n install_requires=requirements,\n extras_require={\n \"docs\": [\"sphinx\", \"sphinxcontrib-napoleon\", \"sphinx_rtd_theme\"],\n },\n cmdclass={\n \"install\": InstallCmd,\n \"develop\": DevelopCmd\n })\n" }, { "path": "srv/GQCNNGraspPlanner.srv", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n# request params\nsensor_msgs/Image color_image\nsensor_msgs/Image depth_image\nsensor_msgs/CameraInfo camera_info\n---\n# response params\nGQCNNGrasp grasp\n" }, { "path": "srv/GQCNNGraspPlannerBoundingBox.srv", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n# request params\nsensor_msgs/Image color_image\nsensor_msgs/Image depth_image\nsensor_msgs/CameraInfo camera_info\nBoundingBox bounding_box\n---\n# response params\nGQCNNGrasp grasp\n" }, { "path": "srv/GQCNNGraspPlannerFull.srv", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n# request params\nsensor_msgs/Image color_image\nsensor_msgs/Image depth_image\nsensor_msgs/CameraInfo camera_info\nBoundingBox bounding_box\nsensor_msgs/Image segmask\n---\n# response params\nGQCNNGrasp grasp\n" }, { "path": "srv/GQCNNGraspPlannerSegmask.srv", "content": "# Copyright ©2017. The Regents of the University of California (Regents).\n# All Rights Reserved. Permission to use, copy, modify, and distribute this\n# software and its documentation for educational, research, and not-for-profit\n# purposes, without fee and without a signed licensing agreement, is hereby\n# granted, provided that the above copyright notice, this paragraph and the\n# following two paragraphs appear in all copies, modifications, and\n# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\n# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\n# otl@berkeley.edu,\n# http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\n# IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\n# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\n# THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\n# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\n# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\n# HEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\n# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n# request params\nsensor_msgs/Image color_image\nsensor_msgs/Image depth_image\nsensor_msgs/CameraInfo camera_info\nsensor_msgs/Image segmask\n---\n# response params\nGQCNNGrasp grasp\n" }, { "path": "tools/analyze_gqcnn_performance.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nAnalyzes a GQ-CNN model.\n\nAuthor\n------\nVishal Satish & Jeff Mahler\n\"\"\"\nimport argparse\nimport os\n\nfrom autolab_core import YamlConfig, Logger\nfrom gqcnn import GQCNNAnalyzer\n\n# Setup logger.\nlogger = Logger.get_logger(\"tools/analyze_gqcnn_performance.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(\n description=(\"Analyze a Grasp Quality Convolutional Neural Network\"\n \" with TensorFlow\"))\n parser.add_argument(\"model_name\",\n type=str,\n default=None,\n help=\"name of model to analyze\")\n parser.add_argument(\"--output_dir\",\n type=str,\n default=None,\n help=\"path to save the analysis\")\n parser.add_argument(\n \"--dataset_config_filename\",\n type=str,\n default=None,\n help=\"path to a configuration file for testing on a custom dataset\")\n parser.add_argument(\"--config_filename\",\n type=str,\n default=None,\n help=\"path to the configuration file to use\")\n parser.add_argument(\"--model_dir\",\n type=str,\n default=None,\n help=\"path to the model\")\n args = parser.parse_args()\n model_name = args.model_name\n output_dir = args.output_dir\n dataset_config_filename = args.dataset_config_filename\n config_filename = args.config_filename\n model_dir = args.model_dir\n\n # Create model dir.\n if model_dir is None:\n model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"../models\")\n model_dir = os.path.join(model_dir, model_name)\n\n # If `model_dir` contains many models, analyze all of them.\n model_dir = [model_dir]\n if \"config.json\" not in os.listdir(model_dir[0]):\n logger.warning(\n \"Found multiple models in model_dir, analyzing all of them...\")\n models = os.listdir(model_dir[0])\n model_dir = [os.path.join(model_dir[0], model) for model in models]\n\n # Set defaults.\n if output_dir is None:\n output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"../analysis\")\n if config_filename is None:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/tools/analyze_gqcnn_performance.yaml\")\n\n # Turn relative paths absolute.\n if not os.path.isabs(output_dir):\n output_dir = os.path.join(os.getcwd(), output_dir)\n if not os.path.isabs(config_filename):\n config_filename = os.path.join(os.getcwd(), config_filename)\n if dataset_config_filename is not None and not os.path.isabs(\n dataset_config_filename):\n dataset_config_filename = os.path.join(os.getcwd(),\n dataset_config_filename)\n\n # Make the output dir.\n if not os.path.exists(output_dir):\n os.mkdir(output_dir)\n\n # Read config.\n config = YamlConfig(config_filename)\n\n dataset_config = None\n if dataset_config_filename is not None:\n dataset_config = YamlConfig(dataset_config_filename)\n\n # Run the analyzer.\n analyzer = GQCNNAnalyzer(config, plot_backend=\"pdf\")\n for model in model_dir:\n analyzer.analyze(model, output_dir, dataset_config)\n" }, { "path": "tools/finetune.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nScript for fine-tuning a Grasp Quality Convolutional Neural Network (GQ-CNN).\n\nAuthor\n------\nVishal Satish & Jeff Mahler\n\"\"\"\nimport argparse\nimport os\nimport time\n\nfrom autolab_core import YamlConfig, Logger\nimport autolab_core.utils as utils\nfrom gqcnn import get_gqcnn_model, get_gqcnn_trainer\nfrom gqcnn import utils as gqcnn_utils\n\n# Setup logger.\nlogger = Logger.get_logger(\"tools/finetune.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(description=(\n \"Fine-Tune a pre-trained Grasp Quality Convolutional Neural Network\"\n \" with TensorFlow\"))\n parser.add_argument(\n \"dataset_dir\",\n type=str,\n default=None,\n help=\"path to the dataset to use for training and validation\")\n parser.add_argument(\"base_model_name\",\n type=str,\n default=None,\n help=\"name of the pre-trained model to fine-tune\")\n parser.add_argument(\"--split_name\",\n type=str,\n default=\"image_wise\",\n help=\"name of the split to train on\")\n parser.add_argument(\"--output_dir\",\n type=str,\n default=None,\n help=\"path to store the model\")\n parser.add_argument(\"--tensorboard_port\",\n type=int,\n default=None,\n help=\"port to launch tensorboard on\")\n parser.add_argument(\"--seed\",\n type=int,\n default=None,\n help=\"random seed for training\")\n parser.add_argument(\"--config_filename\",\n type=str,\n default=None,\n help=\"path to the configuration file to use\")\n parser.add_argument(\"--model_dir\",\n type=str,\n default=None,\n help=\"path to the pre-trained model to fine-tune\")\n parser.add_argument(\"--name\",\n type=str,\n default=None,\n help=\"name for the trained model\")\n parser.add_argument(\n \"--save_datetime\",\n type=bool,\n default=False,\n help=(\"whether or not to save a model with the date and time of\"\n \" training\"))\n parser.add_argument(\"--backend\",\n type=str,\n default=\"tf\",\n help=\"the deep learning framework to use\")\n args = parser.parse_args()\n dataset_dir = args.dataset_dir\n base_model_name = args.base_model_name\n split_name = args.split_name\n output_dir = args.output_dir\n tensorboard_port = args.tensorboard_port\n seed = args.seed\n config_filename = args.config_filename\n model_dir = args.model_dir\n name = args.name\n save_datetime = args.save_datetime\n backend = args.backend\n\n # Set default output dir.\n if output_dir is None:\n output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"../models\")\n\n # Set default config filename.\n if config_filename is None:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/finetune.yaml\")\n\n # Set default model dir.\n if model_dir is None:\n model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"../models\")\n\n # Turn relative paths absolute.\n if not os.path.isabs(dataset_dir):\n dataset_dir = os.path.join(os.getcwd(), dataset_dir)\n if not os.path.isabs(model_dir):\n model_dir = os.path.join(os.getcwd(), model_dir)\n if not os.path.isabs(output_dir):\n output_dir = os.path.join(os.getcwd(), output_dir)\n if not os.path.isabs(config_filename):\n config_filename = os.path.join(os.getcwd(), config_filename)\n\n # Create full path to the pre-trained model.\n model_dir = os.path.join(model_dir, base_model_name)\n\n # Create output dir if necessary.\n utils.mkdir_safe(output_dir)\n\n # Open train config.\n train_config = YamlConfig(config_filename)\n if seed is not None:\n train_config[\"seed\"] = seed\n train_config[\"gqcnn\"][\"seed\"] = seed\n if tensorboard_port is not None:\n train_config[\"tensorboard_port\"] = tensorboard_port\n gqcnn_params = train_config[\"gqcnn\"]\n\n # Create a unique output folder based on the date and time.\n if save_datetime:\n # Create output dir.\n unique_name = time.strftime(\"%Y%m%d-%H%M%S\")\n output_dir = os.path.join(output_dir, unique_name)\n utils.mkdir_safe(output_dir)\n\n # Set visible devices.\n if \"gpu_list\" in train_config:\n gqcnn_utils.set_cuda_visible_devices(train_config[\"gpu_list\"])\n\n # Fine-tune the network.\n start_time = time.time()\n gqcnn = get_gqcnn_model(backend)(gqcnn_params)\n trainer = get_gqcnn_trainer(backend)(gqcnn,\n dataset_dir,\n split_name,\n output_dir,\n train_config,\n name=name)\n trainer.finetune(model_dir)\n logger.info(\"Total Fine-tuning Time: \" +\n str(utils.get_elapsed_time(time.time() - start_time)))\n" }, { "path": "tools/hyperparam_search.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nScript for searching over Grasp Quality Convolutional Neural Network (GQ-CNN)\nhyper-parameters.\n\nAuthor\n------\nVishal Satish\n\"\"\"\nimport argparse\n\nfrom autolab_core import YamlConfig, Logger\nfrom gqcnn import GQCNNSearch\n\n# Set up logger.\nlogger = Logger.get_logger(\"tools/hyperparam_search.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(\n description=\"Hyper-parameter search for GQ-CNN.\")\n parser.add_argument(\"datasets\",\n nargs=\"+\",\n default=None,\n help=\"path to datasets\")\n parser.add_argument(\"--base_model_dirs\",\n nargs=\"+\",\n default=[],\n help=\"path to pre-trained base models for fine-tuning\")\n parser.add_argument(\"--train_configs\",\n nargs=\"+\",\n default=[\"cfg/train.yaml\"],\n help=\"path to training configs\")\n parser.add_argument(\"--analysis_config\",\n type=str,\n default=\"cfg/tools/analyze_gqcnn_performance.yaml\")\n parser.add_argument(\"--split_names\",\n nargs=\"+\",\n default=[\"image_wise\"],\n help=\"dataset splits to use\")\n parser.add_argument(\"--output_dir\",\n type=str,\n default=\"models\",\n help=\"path to store search data\")\n parser.add_argument(\"--search_name\",\n type=str,\n default=None,\n help=\"name of search\")\n parser.add_argument(\"--cpu_cores\",\n nargs=\"+\",\n default=[],\n help=\"CPU cores to use\")\n parser.add_argument(\"--gpu_devices\",\n nargs=\"+\",\n default=[],\n help=\"GPU devices to use\")\n args = parser.parse_args()\n datasets = args.datasets\n base_model_dirs = args.base_model_dirs\n train_configs = args.train_configs\n analysis_config = args.analysis_config\n split_names = args.split_names\n output_dir = args.output_dir\n search_name = args.search_name\n cpu_cores = [int(core) for core in args.cpu_cores]\n gpu_devices = [int(device) for device in args.gpu_devices]\n\n assert len(datasets) == len(\n train_configs\n ), \"Must have same number of datasets as training configs!\"\n if len(base_model_dirs) > 0:\n models_datasets_mismatch_msg = (\"Must have same number of base models\"\n \" for fine-tuning as datasets and\"\n \" training configs!\")\n assert len(base_model_dirs) == len(\n datasets), models_datasets_mismatch_msg\n if len(split_names) < len(datasets):\n if len(split_names) == 1:\n logger.warning(\n \"Using split '{}' for all datasets/configs...\".format(\n split_names[0]))\n split_names *= len(datasets)\n else:\n not_enough_splits_msg = (\"Can't have fewer splits that\"\n \" datasets/configs provided unless there\"\n \" is only one.\")\n raise ValueError(not_enough_splits_msg)\n\n # Parse configs.\n analysis_config = YamlConfig(analysis_config)\n train_configs = [YamlConfig(cfg) for cfg in train_configs]\n\n # Search.\n search = GQCNNSearch(analysis_config,\n train_configs,\n datasets,\n split_names,\n output_dir=output_dir,\n search_name=search_name,\n cpu_cores=cpu_cores,\n gpu_devices=gpu_devices,\n base_models=base_model_dirs)\n search.search()\n" }, { "path": "tools/plot_training_losses.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nScript to plot the various errors saved during training.\n\nAuthor\n------\nJeff Mahler\n\nRequired Parameters\n-------------------\nmodel_dir : str\n Command line argument, the path to the model whose errors are to plotted.\n All plots and other metrics will be saved to this directory.\n\"\"\"\nimport os\nimport sys\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nfrom autolab_core import Logger\nfrom gqcnn.utils import GeneralConstants, GQCNNFilenames\n\n# Set up logger.\nlogger = Logger.get_logger(\"tools/plot_training_losses.py\")\n\nif __name__ == \"__main__\":\n result_dir = sys.argv[1]\n train_errors_filename = os.path.join(result_dir,\n GQCNNFilenames.TRAIN_ERRORS)\n val_errors_filename = os.path.join(result_dir, GQCNNFilenames.VAL_ERRORS)\n train_iters_filename = os.path.join(result_dir, GQCNNFilenames.TRAIN_ITERS)\n val_iters_filename = os.path.join(result_dir, GQCNNFilenames.VAL_ITERS)\n pct_pos_val_filename = os.path.join(result_dir, GQCNNFilenames.PCT_POS_VAL)\n train_losses_filename = os.path.join(result_dir, GQCNNFilenames.TRAIN_LOSS)\n val_losses_filename = os.path.join(result_dir, GQCNNFilenames.VAL_LOSS)\n\n raw_train_errors = np.load(train_errors_filename)\n val_errors = np.load(val_errors_filename)\n raw_train_iters = np.load(train_iters_filename)\n val_iters = np.load(val_iters_filename)\n pct_pos_val = float(val_errors[0])\n if os.path.exists(pct_pos_val_filename):\n pct_pos_val = 100.0 * np.load(pct_pos_val_filename)\n raw_train_losses = np.load(train_losses_filename)\n val_losses = None\n try:\n val_losses = np.load(val_losses_filename)\n except FileNotFoundError:\n pass\n\n val_errors = np.r_[pct_pos_val, val_errors]\n val_iters = np.r_[0, val_iters]\n\n # Window the training error.\n i = 0\n train_errors = []\n train_losses = []\n train_iters = []\n while i < raw_train_errors.shape[0]:\n train_errors.append(\n np.mean(raw_train_errors[i:i + GeneralConstants.WINDOW]))\n train_losses.append(\n np.mean(raw_train_losses[i:i + GeneralConstants.WINDOW]))\n train_iters.append(i)\n i += GeneralConstants.WINDOW\n train_errors = np.array(train_errors)\n train_losses = np.array(train_losses)\n train_iters = np.array(train_iters)\n\n if val_losses is not None:\n val_losses = np.r_[train_losses[0], val_losses]\n\n init_val_error = val_errors[0]\n norm_train_errors = train_errors / init_val_error\n norm_val_errors = val_errors / init_val_error\n norm_final_val_error = val_errors[-1] / val_errors[0]\n if pct_pos_val > 0:\n norm_final_val_error = val_errors[-1] / pct_pos_val\n\n logger.info(\"TRAIN\")\n logger.info(\"Original Error {}\".format(train_errors[0]))\n logger.info(\"Final Error {}\".format(train_errors[-1]))\n logger.info(\"Orig loss {}\".format(train_losses[0]))\n logger.info(\"Final loss {}\".format(train_losses[-1]))\n\n logger.info(\"VAL\")\n logger.info(\"Original error {}\".format(pct_pos_val))\n logger.info(\"Final error {}\".format(val_errors[-1]))\n logger.info(\"Normalized error {}\".format(norm_final_val_error))\n if val_losses is not None:\n logger.info(\"Orig loss {}\".format(val_losses[0]))\n logger.info(\"Final loss {}\".format(val_losses[-1]))\n\n plt.figure()\n plt.plot(train_iters, train_errors, linewidth=4, color=\"b\")\n plt.plot(val_iters, val_errors, linewidth=4, color=\"g\")\n plt.ylim(0, 100)\n plt.legend((\"Training (Minibatch)\", \"Validation\"), fontsize=15, loc=\"best\")\n plt.xlabel(\"Iteration\", fontsize=15)\n plt.ylabel(\"Error Rate\", fontsize=15)\n\n plt.figure()\n plt.plot(train_iters, norm_train_errors, linewidth=4, color=\"b\")\n plt.plot(val_iters, norm_val_errors, linewidth=4, color=\"g\")\n plt.ylim(0, 2.0)\n plt.legend((\"Training (Minibatch)\", \"Validation\"), fontsize=15, loc=\"best\")\n plt.xlabel(\"Iteration\", fontsize=15)\n plt.ylabel(\"Normalized Error Rate\", fontsize=15)\n\n train_losses[train_losses > 100.0] = 3.0\n plt.figure()\n plt.plot(train_iters, train_losses, linewidth=4, color=\"b\")\n plt.ylim(0, 2.0)\n plt.xlabel(\"Iteration\", fontsize=15)\n plt.ylabel(\"Training Loss\", fontsize=15)\n\n if val_losses is not None:\n val_losses[val_losses > 100.0] = 3.0\n plt.figure()\n plt.plot(val_iters, val_losses, linewidth=4, color=\"b\")\n plt.ylim(0, 2.0)\n plt.xlabel(\"Iteration\", fontsize=15)\n plt.ylabel(\"Validation Loss\", fontsize=15)\n plt.show()\n\n plt.figure(figsize=(8, 6))\n plt.plot(train_iters, train_errors, linewidth=4, color=\"b\")\n plt.plot(val_iters, val_errors, linewidth=4, color=\"g\")\n plt.ylim(0, 100)\n plt.legend((\"Training (Minibatch)\", \"Validation\"), fontsize=15, loc=\"best\")\n plt.xlabel(\"Iteration\", fontsize=15)\n plt.ylabel(\"Error Rate\", fontsize=15)\n plt.savefig(os.path.join(result_dir, \"training_curve.jpg\"))\n\n plt.figure(figsize=(8, 6))\n plt.plot(train_iters, norm_train_errors, linewidth=4, color=\"b\")\n plt.plot(val_iters, norm_val_errors, linewidth=4, color=\"g\")\n plt.ylim(0, 2.0)\n plt.legend((\"Training (Minibatch)\", \"Validation\"), fontsize=15, loc=\"best\")\n plt.xlabel(\"Iteration\", fontsize=15)\n plt.ylabel(\"Normalized Error Rate\", fontsize=15)\n plt.savefig(os.path.join(result_dir, \"normalized_training_curve.jpg\"))\n" }, { "path": "tools/run_policy.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nScript to run saved policy output from user.\n\nAuthor\n------\nJeff Mahler\n\"\"\"\nimport argparse\nimport os\nimport random\nimport time\n\nimport numpy as np\n\nfrom autolab_core import YamlConfig, Logger\nfrom visualization import Visualizer2D as vis2d\nfrom gqcnn import RgbdImageState, ParallelJawGrasp\nfrom gqcnn import CrossEntropyRobustGraspingPolicy\n\n# Set up logger.\nlogger = Logger.get_logger(\"tools/run_policy.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(\n description=(\"Run a saved test case through a GQ-CNN policy. For\"\n \" debugging purposes only.\"))\n parser.add_argument(\"test_case_path\",\n type=str,\n default=None,\n help=\"path to test case\")\n parser.add_argument(\"--config_filename\",\n type=str,\n default=\"cfg/tools/run_policy.yaml\",\n help=\"path to configuration file to use\")\n parser.add_argument(\"--output_dir\",\n type=str,\n default=None,\n help=\"directory to store output\")\n parser.add_argument(\"--seed\", type=int, default=None, help=\"random seed\")\n args = parser.parse_args()\n test_case_path = args.test_case_path\n config_filename = args.config_filename\n output_dir = args.output_dir\n seed = args.seed\n\n # Make output dir.\n if output_dir is not None and not os.path.exists(output_dir):\n os.mkdir(output_dir)\n\n # Make relative paths absolute.\n if not os.path.isabs(config_filename):\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\", config_filename)\n\n # Set random seed.\n if seed is not None:\n np.random.seed(seed)\n random.seed(seed)\n\n # Read config.\n config = YamlConfig(config_filename)\n policy_config = config[\"policy\"]\n\n # Load test case.\n state_path = os.path.join(test_case_path, \"state\")\n action_path = os.path.join(test_case_path, \"action\")\n state = RgbdImageState.load(state_path)\n original_action = ParallelJawGrasp.load(action_path)\n\n # Init policy.\n policy = CrossEntropyRobustGraspingPolicy(policy_config)\n\n if policy_config[\"vis\"][\"input_images\"]:\n vis2d.figure()\n if state.segmask is None:\n vis2d.subplot(1, 2, 1)\n vis2d.imshow(state.rgbd_im.color)\n vis2d.title(\"COLOR\")\n vis2d.subplot(1, 2, 2)\n vis2d.imshow(state.rgbd_im.depth,\n vmin=policy_config[\"vis\"][\"vmin\"],\n vmax=policy_config[\"vis\"][\"vmax\"])\n vis2d.title(\"DEPTH\")\n else:\n vis2d.subplot(1, 3, 1)\n vis2d.imshow(state.rgbd_im.color)\n vis2d.title(\"COLOR\")\n vis2d.subplot(1, 3, 2)\n vis2d.imshow(state.rgbd_im.depth,\n vmin=policy_config[\"vis\"][\"vmin\"],\n vmax=policy_config[\"vis\"][\"vmax\"])\n vis2d.title(\"DEPTH\")\n vis2d.subplot(1, 3, 3)\n vis2d.imshow(state.segmask)\n vis2d.title(\"SEGMASK\")\n filename = None\n if output_dir is not None:\n filename = os.path.join(output_dir, \"input_images.png\")\n vis2d.show(filename)\n\n # Query policy.\n policy_start = time.time()\n action = policy(state)\n logger.info(\"Planning took %.3f sec\" % (time.time() - policy_start))\n\n # Vis final grasp.\n if policy_config[\"vis\"][\"final_grasp\"]:\n vis2d.figure(size=(10, 10))\n vis2d.subplot(1, 2, 1)\n vis2d.imshow(state.rgbd_im.depth,\n vmin=policy_config[\"vis\"][\"vmin\"],\n vmax=policy_config[\"vis\"][\"vmax\"])\n vis2d.grasp(original_action.grasp,\n scale=policy_config[\"vis\"][\"grasp_scale\"],\n show_center=False,\n show_axis=True,\n color=\"r\")\n vis2d.title(\"Original (Q=%.3f) (Z=%.3f)\" %\n (original_action.q_value, original_action.grasp.depth))\n vis2d.subplot(1, 2, 2)\n vis2d.imshow(state.rgbd_im.depth,\n vmin=policy_config[\"vis\"][\"vmin\"],\n vmax=policy_config[\"vis\"][\"vmax\"])\n vis2d.grasp(action.grasp,\n scale=policy_config[\"vis\"][\"grasp_scale\"],\n show_center=False,\n show_axis=True,\n color=\"r\")\n vis2d.title(\"New (Q=%.3f) (Z=%.3f)\" %\n (action.q_value, action.grasp.depth))\n filename = None\n if output_dir is not None:\n filename = os.path.join(output_dir, \"planned_grasp.png\")\n vis2d.show(filename)\n" }, { "path": "tools/train.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCopyright ©2017. The Regents of the University of California (Regents).\nAll Rights Reserved. Permission to use, copy, modify, and distribute this\nsoftware and its documentation for educational, research, and not-for-profit\npurposes, without fee and without a signed licensing agreement, is hereby\ngranted, provided that the above copyright notice, this paragraph and the\nfollowing two paragraphs appear in all copies, modifications, and\ndistributions. Contact The Office of Technology Licensing, UC Berkeley, 2150\nShattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,\notl@berkeley.edu,\nhttp://ipira.berkeley.edu/industry-info for commercial licensing opportunities.\n\nIN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,\nINCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF\nTHE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nREGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED\nHEREUNDER IS PROVIDED \"AS IS\". REGENTS HAS NO OBLIGATION TO PROVIDE\nMAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\nScript for training a Grasp Quality Convolutional Neural Network (GQ-CNN).\n\nAuthor\n------\nVishal Satish & Jeff Mahler\n\"\"\"\nimport argparse\nimport os\nimport time\n\nfrom autolab_core import YamlConfig, Logger\nimport autolab_core.utils as utils\nfrom gqcnn import get_gqcnn_model, get_gqcnn_trainer, utils as gqcnn_utils\n\n# Setup logger.\nlogger = Logger.get_logger(\"tools/train.py\")\n\nif __name__ == \"__main__\":\n # Parse args.\n parser = argparse.ArgumentParser(\n description=(\"Train a Grasp Quality Convolutional Neural Network with\"\n \" TensorFlow\"))\n parser.add_argument(\n \"dataset_dir\",\n type=str,\n default=None,\n help=\"path to the dataset to use for training and validation\")\n parser.add_argument(\"--split_name\",\n type=str,\n default=\"image_wise\",\n help=\"name of the split to train on\")\n parser.add_argument(\"--output_dir\",\n type=str,\n default=None,\n help=\"path to store the model\")\n parser.add_argument(\"--tensorboard_port\",\n type=int,\n default=None,\n help=\"port to launch tensorboard on\")\n parser.add_argument(\"--seed\",\n type=int,\n default=None,\n help=\"random seed for training\")\n parser.add_argument(\"--config_filename\",\n type=str,\n default=None,\n help=\"path to the configuration file to use\")\n parser.add_argument(\"--name\",\n type=str,\n default=None,\n help=\"name for the trained model\")\n parser.add_argument(\n \"--save_datetime\",\n type=bool,\n default=False,\n help=(\"whether or not to save a model with the date and time of\"\n \" training\"))\n parser.add_argument(\"--backend\",\n type=str,\n default=\"tf\",\n help=\"the deep learning framework to use\")\n args = parser.parse_args()\n dataset_dir = args.dataset_dir\n split_name = args.split_name\n output_dir = args.output_dir\n tensorboard_port = args.tensorboard_port\n seed = args.seed\n config_filename = args.config_filename\n name = args.name\n save_datetime = args.save_datetime\n backend = args.backend\n\n # Set default output dir.\n if output_dir is None:\n output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"../models\")\n\n # Set default config filename.\n if config_filename is None:\n config_filename = os.path.join(\n os.path.dirname(os.path.realpath(__file__)), \"..\",\n \"cfg/train.yaml\")\n\n # Turn relative paths absolute.\n if not os.path.isabs(dataset_dir):\n dataset_dir = os.path.join(os.getcwd(), dataset_dir)\n if not os.path.isabs(output_dir):\n output_dir = os.path.join(os.getcwd(), output_dir)\n if not os.path.isabs(config_filename):\n config_filename = os.path.join(os.getcwd(), config_filename)\n\n # Create output dir if necessary.\n utils.mkdir_safe(output_dir)\n\n # Open train config.\n train_config = YamlConfig(config_filename)\n if seed is not None:\n train_config[\"seed\"] = seed\n train_config[\"gqcnn\"][\"seed\"] = seed\n if tensorboard_port is not None:\n train_config[\"tensorboard_port\"] = tensorboard_port\n gqcnn_params = train_config[\"gqcnn\"]\n\n # Create a unique output folder based on the date and time.\n if save_datetime:\n # Create output dir.\n unique_name = time.strftime(\"%Y%m%d-%H%M%S\")\n output_dir = os.path.join(output_dir, unique_name)\n utils.mkdir_safe(output_dir)\n\n # Set visible devices.\n if \"gpu_list\" in train_config:\n gqcnn_utils.set_cuda_visible_devices(train_config[\"gpu_list\"])\n\n # Train the network.\n start_time = time.time()\n gqcnn = get_gqcnn_model(backend)(gqcnn_params)\n trainer = get_gqcnn_trainer(backend)(gqcnn,\n dataset_dir,\n split_name,\n output_dir,\n train_config,\n name=name)\n trainer.train()\n logger.info(\"Total Training Time: \" +\n str(utils.get_elapsed_time(time.time() - start_time)))\n" } ]