[ { "path": ".bazelrc", "content": "build --cxxopt=-std=c++1z\nbuild --incompatible_remove_native_http_archive=false \nbuild --incompatible_package_name_is_a_function=false\n" }, { "path": ".clang-format", "content": "#BasedOnStyle: Google\n\nLanguage: Cpp\n# BasedOnStyle: LLVM\nAccessModifierOffset: -2\nAlignAfterOpenBracket: Align\nAlignConsecutiveAssignments: false\nAlignConsecutiveDeclarations: false\nAlignEscapedNewlinesLeft: true\nAlignOperands: true\nAlignTrailingComments: false\nAllowAllParametersOfDeclarationOnNextLine: true\nAllowShortBlocksOnASingleLine: false\nAllowShortCaseLabelsOnASingleLine: false\nAllowShortIfStatementsOnASingleLine: true\nAllowShortFunctionsOnASingleLine: true\nAllowShortLoopsOnASingleLine: true\nAlwaysBreakAfterDefinitionReturnType: None\nAlwaysBreakAfterReturnType: None\nAlwaysBreakBeforeMultilineStrings: false\nAlwaysBreakTemplateDeclarations: true\nBinPackArguments: true\nBinPackParameters: true\nBraceWrapping:\n AfterClass: true\n AfterControlStatement: true\n AfterEnum: true\n AfterFunction: true\n AfterNamespace: false\n AfterObjCDeclaration: false\n AfterStruct: true\n AfterUnion: true\n BeforeCatch: true\n BeforeElse: true\n IndentBraces: false\nBreakBeforeBinaryOperators: None\nBreakBeforeBraces: Custom\nBreakBeforeTernaryOperators: true\nBreakConstructorInitializersBeforeComma: false\nBreakAfterJavaFieldAnnotations: false\nBreakStringLiterals: true\nColumnLimit: 100\nCommentPragmas: '^ IWYU pragma:'\nConstructorInitializerAllOnOneLineOrOnePerLine: false\nConstructorInitializerIndentWidth: 4\nContinuationIndentWidth: 4\nCpp11BracedListStyle: true\nDerivePointerAlignment: false\nDisableFormat: false\nForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ]\nIncludeCategories:\n - Regex: '^\"(llvm|llvm-c|clang|clang-c)/'\n Priority: 2\n - Regex: '^(<|\"(gtest|isl|json)/)'\n Priority: 3\n - Regex: '.*'\n Priority: 1\nIncludeIsMainRegex: '$'\nIndentCaseLabels: true\nIndentWidth: 4\nIndentWrappedFunctionNames: true\nJavaScriptQuotes: Leave\nKeepEmptyLinesAtTheStartOfBlocks: false\nMacroBlockBegin: ''\nMacroBlockEnd: ''\nMaxEmptyLinesToKeep: 1\nNamespaceIndentation: None\nPenaltyBreakBeforeFirstCallParameter: 19\nPenaltyBreakComment: 300\nPenaltyBreakFirstLessLess: 100\nPenaltyBreakString: 1000\nPenaltyExcessCharacter: 1000000\nPenaltyReturnTypeOnItsOwnLine: 60\nPointerAlignment: Left\nReflowComments: true\nSortIncludes: true\nSpaceAfterCStyleCast: false\nSpaceAfterTemplateKeyword: false\nSpaceBeforeAssignmentOperators: true\nSpaceBeforeParens: Never\nSpaceInEmptyParentheses: false\nSpacesBeforeTrailingComments: 1\nSpacesInAngles: false\nSpacesInContainerLiterals: false\nSpacesInCStyleCastParentheses: false\nSpacesInParentheses: false\nSpacesInSquareBrackets: false\nStandard: Cpp11\nTabWidth: 4\nUseTab: Never\n" }, { "path": ".dockerignore", "content": "build\n*.engine\nmodels\n@eaDir\n__pycache__\nbazel-*\n" }, { "path": ".gitmodules", "content": "[submodule \"third_party/cpuaff\"]\n\tpath = third_party/cpuaff\n\turl = https://github.com/dcdillon/cpuaff\n[submodule \"third_party/gflags\"]\n\tpath = third_party/gflags\n\turl = https://github.com/gflags/gflags.git\n[submodule \"third_party/glog\"]\n\tpath = third_party/glog\n\turl = https://github.com/google/glog.git\n[submodule \"third_party/grpc\"]\n\tpath = third_party/grpc\n\turl = https://github.com/grpc/grpc\n[submodule \"third_party/wait-for-it\"]\n\tpath = third_party/wait-for-it\n\turl = https://github.com/vishnubob/wait-for-it\n[submodule \"third_party/benchmark\"]\n\tpath = third_party/benchmark\n\turl = https://github.com/google/benchmark.git\n[submodule \"third_party/googletest\"]\n\tpath = third_party/googletest\n\turl = https://github.com/google/googletest.git\n[submodule \"third_party/pybind11\"]\n\tpath = third_party/pybind11\n\turl = https://github.com/pybind/pybind11.git\n[submodule \"third_party/flatbuffers\"]\n\tpath = third_party/flatbuffers\n\turl = https://github.com/google/flatbuffers.git\n" }, { "path": "BUILD.bazel", "content": "package(default_visibility = [\"//visibility:public\"])\n" }, { "path": "CLA", "content": " The NVIDIA TensorRT Laboratory\n Software Grant and Corporate Contributor License Agreement (\"Agreement\")\n\n Thank you for your interest in the NVIDIA TensorRT Laboratory Project \n (the \"Project\"). In order to clarify the intellectual property license\n granted with Contributions from any person or entity, NVIDIA\n Corporation (the “Copyright Holders\") must have a Contributor License\n Agreement (CLA) on file that has been signed by each Contributor,\n indicating agreement to the license terms below. This license is\n for your protection as a Contributor as well as the protection of the\n Project and its users; it does not change your rights to use your own\n Contributions for any other purpose.\n\n This version of the Agreement allows an entity (the \"Corporation\") to\n submit Contributions to the Project, to authorize Contributions\n submitted by its designated employees to the Project, and to grant\n copyright and patent licenses thereto to the Copyright Holders.\n\n If you have not already done so, please complete and sign, then scan and\n email a pdf file of this Agreement to rolson@nvidia.com.\n Please read this document carefully before signing and keep a copy for\n your records.\n\n Corporation name: ________________________________________________\n\n Corporation address: ________________________________________________\n\n ________________________________________________\n\n ________________________________________________\n\n Point of Contact: ________________________________________________\n\n E-Mail: ________________________________________________\n\n Telephone: _____________________ Fax: _____________________\n\n\n You accept and agree to the following terms and conditions for Your\n present and future Contributions submitted to the Project. In\n return, the Copyright Holders shall not use Your Contributions in a way\n that is contrary to the public benefit or inconsistent with its nonprofit\n status and bylaws in effect at the time of the Contribution. Except\n for the license granted herein to the Copyright Holders and recipients of\n software distributed by the Copyright Holders, You reserve all right, title,\n and interest in and to Your Contributions.\n\n 1. Definitions.\n\n \"You\" (or \"Your\") shall mean the copyright owner or legal entity\n authorized by the copyright owner that is making this Agreement\n with the Copyright Holders. For legal entities, the entity making a\n Contribution and all other entities that control, are controlled by,\n or are under common control with that entity are considered to be a\n single Contributor. For the purposes of this definition, \"control\"\n means (i) the power, direct or indirect, to cause the direction or\n management of such entity, whether by contract or otherwise, or\n (ii) ownership of fifty percent (50%) or more of the outstanding\n shares, or (iii) beneficial ownership of such entity.\n\n \"Contribution\" shall mean the code, documentation or other original\n works of authorship expressly identified in Schedule B, as well as\n any original work of authorship, including\n any modifications or additions to an existing work, that is intentionally\n submitted by You to the Copyright Holders for inclusion in, or\n documentation of, any of the products owned or managed by the\n Copyright Holders (the \"Work\"). For the purposes of this definition,\n \"submitted\" means any form of electronic, verbal, or written\n communication sent to the Copyright Holders or its representatives,\n including but not limited to communication on electronic mailing\n lists, source code control systems, and issue tracking systems\n that are managed by, or on behalf of, the Copyright Holders for the\n purpose of discussing and improving the Work, but excluding\n communication that is conspicuously marked or otherwise designated\n in writing by You as \"Not a Contribution.\"\n\n 2. Grant of Copyright License. Subject to the terms and conditions\n of this Agreement, You hereby grant to the Copyright Holders and to\n recipients of software distributed by the Copyright Holders a\n perpetual, worldwide, non-exclusive, no-charge, royalty-free,\n irrevocable copyright license to reproduce, prepare derivative works\n of, publicly display, publicly perform, sublicense, and distribute\n Your Contributions and such derivative works.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this Agreement, You hereby grant to the Copyright Holders and to\n recipients of software distributed by the Copyright Holders\n a perpetual, worldwide, non-exclusive, no-charge, royalty-free,\n irrevocable (except as stated in this section) patent license\n to make, have made, use, offer to sell, sell, import, and otherwise\n transfer the Work, where such license applies only to those\n patent claims licensable by You that are necessarily infringed\n by Your Contribution(s) alone or by combination of Your Contribution(s)\n with the Work to which such Contribution(s) were submitted.\n If any entity institutes patent litigation against You or any\n other entity (including a cross-claim or counterclaim in a lawsuit)\n alleging that your Contribution, or the Work to which you have\n contributed, constitutes direct or contributory patent infringement,\n then any patent licenses granted to that entity under this Agreement\n for that Contribution or Work shall terminate as of the date such\n litigation is filed.\n\n 4. You represent that You are legally entitled to grant the above\n license. You represent further that each employee of the\n Corporation designated on Schedule A below (or in a subsequent\n written modification to that Schedule) is authorized to submit\n Contributions on behalf of the Corporation.\n\n 5. You represent that each of Your Contributions is Your original\n creation (see section 7 for submissions on behalf of others).\n\n 6. You are not expected to provide support for Your Contributions,\n except to the extent You desire to provide support. You may provide\n support for free, for a fee, or not at all. Unless required by\n applicable law or agreed to in writing, You provide Your\n Contributions on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS\n OF ANY KIND, either express or implied, including, without\n limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT,\n MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.\n\n 7. Should You wish to submit work that is not Your original creation,\n You may submit it to the Copyright Holders separately from any\n Contribution, identifying the complete details of its source and\n of any license or other restriction (including, but not limited\n to, related patents, trademarks, and license agreements) of which\n you are personally aware, and conspicuously marking the work as\n \"Submitted on behalf of a third-party: [named here]\".\n\n 8. It is your responsibility to notify the Copyright Holders when any change\n is required to the list of designated employees authorized to submit\n Contributions on behalf of the Corporation, or to the Corporation's\n Point of Contact with the Copyright Holders.\n\n\n\n Please sign: __________________________________ Date: _______________\n\n Title: __________________________________\n\n Corporation: __________________________________\n\n\n\n\nSchedule A\n\n [Initial list of designated employees. NB: authorization is not\n tied to particular Contributions.]\n\n\n\n\n\n\nSchedule B\n\n [Identification of optional concurrent software grant. Would be\n left blank or omitted if there is no concurrent software grant.]\n\n\n" }, { "path": "CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\ncmake_minimum_required(VERSION 3.13 FATAL_ERROR)\n\noption (BUILD_DEPENDENCIES \"Whether or not a superbuild should be invoked\" ON)\noption (BUILD_MEMORY \"Whether or not build trtlab/memory\" ON)\noption (BUILD_CORE \"Whether or not build trtlab/core\" ON)\noption (BUILD_CUDA \"Whether or not build trtlab/cuda\" ON)\noption (BUILD_NVRPC \"Whether or not build trtlab/nvrpc\" ON)\noption (BUILD_TENSORRT \"Whether or not build trtlab/tensorrt\" ON)\noption (BUILD_PYTHON \"Whether or not build trtlab/pybind\" OFF)\noption (BUILD_EXAMPLES \"Whether or not to build trtlab examples\" OFF)\n\nif (BUILD_DEPENDENCIES)\n project (trtlab_dependencies NONE)\n include (cmake/dependencies.cmake)\n return() # stop processing this file further\nelse()\n project (trtlab)\nendif()\n\n# CMake path\nlist(APPEND CMAKE_MODULE_PATH \"${PROJECT_SOURCE_DIR}/cmake\")\nlist(APPEND CMAKE_PREFIX_PATH \"${PROJECT_SOURCE_DIR}/cmake\")\n\n# trtlab \nadd_subdirectory(trtlab)\n\nif (BUILD_EXAMPLES)\n add_subdirectory(examples)\nendif()\n" }, { "path": "CREDITS.md", "content": "`ThreadPool` class was derived from https://github.com/progschj/ThreadPool\n\n> Copyright (c) 2012 Jakob Progsch, Václav Zeman\n> \n> This software is provided 'as-is', without any express or implied\n> warranty. In no event will the authors be held liable for any damages\n> arising from the use of this software.\n> \n> Permission is granted to anyone to use this software for any purpose,\n> including commercial applications, and to alter it and redistribute it\n> freely, subject to the following restrictions:\n> \n> 1. The origin of this software must not be misrepresented; you must not\n> claim that you wrote the original software. If you use this software\n> in a product, an acknowledgment in the product documentation would be\n> appreciated but is not required.\n> \n> 2. Altered source versions must be plainly marked as such, and must not be\n> misrepresented as being the original software.\n> \n> 3. This notice may not be removed or altered from any source\n> distribution.\n> \n> Modifications to the original work include:\n> * Header-only file was split into .h/.cc files\n> * Added an extra safety check (lines 30-31) in the construction (.cc file).\n> * Added CPU affinity options to the constructor\n\n-----\n\n`cpuaff` is distributed unmodified from the original in [`third-party/cpuaff`]\n(https://github.com/dcdillon/cpuaff)\n\n> Copyright (c) 2015, Daniel C. Dillon\n> All rights reserved.\n> \n> Redistribution and use in source and binary forms, with or without\n> modification, are permitted provided that the following conditions are met:\n> \n> * Redistributions of source code must retain the above copyright notice, this\n> list of conditions and the following disclaimer.\n> \n> * Redistributions in binary form must reproduce the above copyright notice,\n> this list of conditions and the following disclaimer in the documentation\n> and/or other materials provided with the distribution.\n> \n> * Neither the name of cpuaff nor the names of its\n> contributors may be used to endorse or promote products derived from\n> this software without specific prior written permission.\n> \n> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n> AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n> IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n> DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\n> FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n> DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n> SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n> CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\n> OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n> OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n-----\n\n`wait-for-it.sh` is distributed unmodified from the original in [`third-pary/wait-for-it`]\n(https://github.com/vishnubob/wait-for-it).\n\n> The MIT License (MIT)\n> Copyright (c) 2016 Giles Hall\n> \n> Permission is hereby granted, free of charge, to any person obtaining a copy of\n> this software and associated documentation files (the \"Software\"), to deal in\n> the Software without restriction, including without limitation the rights to\n> use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\n> of the Software, and to permit persons to whom the Software is furnished to do\n> so, subject to the following conditions:\n> \n> The above copyright notice and this permission notice shall be included in all\n> copies or substantial portions of the Software.\n> \n> THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n\n---\n\nExample gRPC client code was used with modification from the [gRPC project]\n(https://github.com/grpc/grpc), specifically the [synchronous c++ client]\n(https://github.com/grpc/grpc/blob/master/examples/cpp/helloworld/greeter_client.cc)\n\n> Copyright 2015 gRPC authors.\n> \n> Licensed under the Apache License, Version 2.0 (the \"License\");\n> you may not use this file except in compliance with the License.\n> You may obtain a copy of the License at\n> \n> http://www.apache.org/licenses/LICENSE-2.0\n> \n> Unless required by applicable law or agreed to in writing, software\n> distributed under the License is distributed on an \"AS IS\" BASIS,\n> WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n> See the License for the specific language governing permissions and\n> limitations under the License.\n\n---\n\n[moodycamel::ConcurrentQueue](https://github.com/cameron314/concurrentqueue) is\nadded unmodified to the Docker images and loaded into the `playground` namespace.\n\n> Simplified BSD License:\n> \n> Copyright (c) 2013-2016, Cameron Desrochers. All rights reserved.\n> \n> Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n> \n> Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n> Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n---\n\n`transwarp` is used unmodified from the original [bloomen/transwarp](https://github.com/bloomen/transwarp)\n\n> MIT License\n> \n> Copyright (c) 2018-2019 Christian Blume, Guan Wang\n> \n> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n> \n> The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n> \n> THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\n---\n\nCaffe ResNet-50 and ResNet-152 models from [KaimingHe/deep-residual-networks]\n(https://github.com/KaimingHe/deep-residual-networks) are included without modification.\n\n> The MIT License (MIT)\n> \n> Copyright (c) 2016 Shaoqing Ren\n> \n> Permission is hereby granted, free of charge, to any person obtaining a copy\n> of this software and associated documentation files (the \"Software\"), to deal\n> in the Software without restriction, including without limitation the rights\n> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n> copies of the Software, and to permit persons to whom the Software is\n> furnished to do so, subject to the following conditions:\n> \n> The above copyright notice and this permission notice shall be included in all\n> copies or substantial portions of the Software.\n> \n> THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n> SOFTWARE.\n\n---\n" }, { "path": "Dockerfile", "content": "# stage 1 - development container\n# holds the core nvidia libraries but does not container the project source code\n# use this container for development by mapping our source into the image which\n# persists your source code outside of the container lifecycle\n\nFROM nvcr.io/nvidia/tensorrt:20.06-py3 AS base\n\nRUN apt update\nRUN apt install -y clang-format libssl-dev openssl libz-dev software-properties-common\n\n# remove base cmake\nRUN apt remove --purge -y cmake\nRUN apt autoremove -y\nRUN apt autoclean -y\n\n# install cmake ppa from kitware - https://apt.kitware.com/\nRUN apt install -y apt-transport-https ca-certificates gnupg software-properties-common wget\nRUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -\nRUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'\nRUN apt update && apt install -y cmake\n\n# then remove FindGTest.cmake installed by cmake\nRUN find / -name \"FindGTest.cmake\" -exec rm -f {} \\;\n\n# add cufft and nvml to the container image\nRUN apt install -y libcufft-dev-11-0 cuda-nvml-dev-11-0\n\n# override some envs\nENV LD_LIBRARY_PATH=/externals/myelin/x86_64/cuda-11.0/lib:/externals/cudnn/x86_64/8.0/cuda-11.0/lib64:/usr/local/cuda-11.0/targets/x86_64-linux/lib\nENV CCACHE_DIR=/tmp/.ccache\nRUN cd /usr/lib/x86_64-linux-gnu && ln -s libnvidia-ml.so.1 libnvidia-ml.so\n\n\n# stage 2: build the project inside the dev container\n\nFROM base AS trtlab\n\nWORKDIR /work\n\nCOPY . .\n\nRUN mkdir build && cd build && cmake .. && make -j\n" }, { "path": "LICENSE", "content": "BSD 3-Clause License\n\nCopyright (c) 2018-2019, NVIDIA Corporation\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n* Neither the name of the copyright holder nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "README.md", "content": "# TensorRT Laboratory\n\nThe TensorRT Laboratory (trtlab) is a general purpose set of tools to build customer inference applications\nand services.\n\n[Triton](https://github.com/nvidia/triton) is a professional grade production inference server.\n\nThis project is broken into 4 primary components:\n\n * `memory` is based on [foonathan/memory](https://github.com/foonathan/memory) the `memory` module \n was designed to write custom allocators for both host and gpu memory. Several custom allocators are\n included. \n\n * `core` contains host/cpu-side tools for common components such as thread pools, resource pool, \n and userspace threading based on boost fibers.\n\n * `cuda` extends `memory` with a new memory_type for CUDA device memory. All custom allocators\n in `memory` can be used with `device_memory`, `device_managed_memory` or `host_pinned_memory`.\n\n * `nvrpc` is an abstraction layer for building asynchronous microservices. The current implementation\n is based on grpc.\n\n * `tensorrt` provides an opinionated runtime built on the TensorRT API.\n\n## Quickstart\n\nThe easiest way to manage the external NVIDIA dependencies is to leverage the containers hosted on\n[NGC](https://ngc.nvidia.com). For bare metal installs, use the `Dockerfile` as a template for\nwhich NVIDIA libraries to install.\n\n```\ndocker build -t trtlab . \n```\n\nFor development purposes, the following set of commands first builds the base image, then\nmaps the source code on the host into a running container.\n\n\n```\ndocker build -t trtlab:dev --target base .\ndocker run --rm -ti --gpus=all -v $PWD:/work --workdir=/work --net=host trtlab:dev bash\n```\n\n\n## Copyright and License\n\nThis project is released under the [BSD 3-clause license](LICENSE).\n\n## Issues and Contributing\n\n* Please let us know by [filing a new issue](https://github.com/NVIDIA/tensorrt-laboratory/issues/new)\n* You can contribute by opening a [pull request](https://help.github.com/articles/using-pull-requests/)\n\nPull requests with changes of 10 lines or more will require a [Contributor License Agreement](CLA).\n" }, { "path": "WORKSPACE", "content": "workspace(name = \"com_github_nvidia_trtlab\")\n\nload(\":bazel/repositories.bzl\", \"repositories\")\nrepositories()\n\nload (\"//bazel:cuda_configure.bzl\", \"cuda_configure\")\ncuda_configure(name = \"local_config_cuda\")\n\nload (\"//bazel:tensorrt_configure.bzl\", \"tensorrt_configure\")\ntensorrt_configure(name = \"local_config_tensorrt\")\n\nload(\"@com_github_grpc_grpc//bazel:grpc_deps.bzl\", \"grpc_deps\")\ngrpc_deps()\n" }, { "path": "bazel/BUILD.bazel", "content": "exports_files(\n glob([\"*.bzl\"]),\n visibility = [\"//visibility:public\"],\n)\n" }, { "path": "bazel/cuda_configure.bzl", "content": "\"\"\"Build rule generator for locally installed CUDA toolkit and cuDNN SDK.\"\"\"\n\n# src: https://github.com/google/nvidia_libs_test\n\ndef _get_env_var(repository_ctx, name, default):\n if name in repository_ctx.os.environ:\n return repository_ctx.os.environ[name]\n return default\n\ndef _impl(repository_ctx):\n cuda_path = _get_env_var(repository_ctx, \"CUDA_PATH\", \"/usr/local/cuda\")\n cudnn_path = _get_env_var(repository_ctx, \"CUDNN_PATH\", cuda_path)\n\n print(\"Using CUDA from %s\\n\" % cuda_path)\n print(\"Using cuDNN from %s\\n\" % cudnn_path)\n\n repository_ctx.symlink(cuda_path, \"cuda\")\n repository_ctx.symlink(cudnn_path, \"cudnn\")\n\n repository_ctx.file(\"nvcc.sh\", \"\"\"\n#! /bin/bash\nrepo_path=%s\ncompiler=${CC:+\"--compiler-bindir=$CC\"}\n$repo_path/cuda/bin/nvcc $compiler --compiler-options=-fPIC --include-path=$repo_path $*\n\"\"\" % repository_ctx.path(\".\"))\n\n repository_ctx.file(\"BUILD\", \"\"\"\npackage(default_visibility = [\"//visibility:public\"])\n\nsh_binary(\n name = \"nvcc\",\n srcs = [\"nvcc.sh\"],\n)\n\n# The *_headers cc_library rules below aren't cc_inc_library rules because\n# dependent targets would only see the first one.\n\ncc_library(\n name = \"cuda_headers\",\n hdrs = glob(\n include = [\"cuda/include/**/*.h*\"],\n exclude = [\"cuda/include/cudnn.h\"]\n ),\n # Allows including CUDA headers with angle brackets.\n includes = [\"cuda/include\"],\n)\n\ncc_library(\n name = \"cuda\",\n srcs = [\"cuda/lib64/stubs/libcuda.so\"],\n linkopts = [\"-ldl\"],\n)\n\ncc_library(\n name = \"cuda_runtime\",\n srcs = [\"cuda/lib64/libcudart_static.a\"],\n deps = [\":cuda\"],\n linkopts = [\"-lrt\"],\n)\n\ncc_library(\n name = \"curand_static\",\n srcs = [\n \"cuda/lib64/libcurand_static.a\",\n \"cuda/lib64/libculibos.a\",\n ],\n)\n\ncc_library(\n name = \"cupti_headers\",\n hdrs = glob([\"cuda/extras/CUPTI/include/**/*.h\"]),\n # Allows including CUPTI headers with angle brackets.\n includes = [\"cuda/extras/CUPTI/include\"],\n)\n\ncc_library(\n name = \"cupti\",\n srcs = glob([\"cuda/extras/CUPTI/lib64/libcupti.so*\"]),\n)\n\ncc_library(\n name = \"cudnn\",\n srcs = [\n \"cudnn/lib64/libcudnn_static.a\",\n \"cuda/lib64/libcublas_static.a\",\n \"cuda/lib64/libculibos.a\",\n ],\n hdrs = [\"cudnn/include/cudnn.h\"],\n deps = [\n \":cuda\",\n \":cuda_headers\"\n ],\n)\n\ncc_library(\n name = \"cuda_util\",\n deps = [\":cuda_util_compile\"],\n)\n\"\"\")\n\ncuda_configure = repository_rule(\n implementation = _impl,\n environ = [\"CUDA_PATH\", \"CUDNN_PATH\"],\n)\n" }, { "path": "bazel/repositories.bzl", "content": "load(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\n\ndef repositories():\n _maybe(\n http_archive,\n name = \"com_github_antonovvk_bazel_rules\",\n sha256 = \"ba75b07d3fd297375a6688e9a16583eb616e7a74b3d5e8791e7a222cf36ab26e\",\n strip_prefix = \"bazel_rules-98ddd7e4f7c63ea0868f08bcc228463dac2f9f12\",\n urls = [\n \"https://mirror.bazel.build/github.com/antonovvk/bazel_rules/archive/98ddd7e4f7c63ea0868f08bcc228463dac2f9f12.tar.gz\",\n \"https://github.com/antonovvk/bazel_rules/archive/98ddd7e4f7c63ea0868f08bcc228463dac2f9f12.tar.gz\",\n ],\n )\n\n _maybe(\n http_archive,\n name = \"com_github_gflags_gflags\",\n sha256 = \"6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe\",\n strip_prefix = \"gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a\",\n urls = [\n \"https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz\",\n \"https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz\",\n ],\n )\n\n _maybe(\n http_archive,\n name = \"com_google_glog\",\n sha256 = \"1ee310e5d0a19b9d584a855000434bb724aa744745d5b8ab1855c85bff8a8e21\",\n strip_prefix = \"glog-028d37889a1e80e8a07da1b8945ac706259e5fd8\",\n urls = [\n \"https://mirror.bazel.build/github.com/google/glog/archive/028d37889a1e80e8a07da1b8945ac706259e5fd8.tar.gz\",\n \"https://github.com/google/glog/archive/028d37889a1e80e8a07da1b8945ac706259e5fd8.tar.gz\",\n ],\n )\n\n _maybe(\n http_archive,\n name = \"com_google_googletest\",\n sha256 = \"c18f281fd6621bb264570b99860a0241939b4a251c9b1af709b811d33bc63af8\",\n strip_prefix = \"googletest-e3bd4cbeaeef3cee65a68a8bd3c535cb779e9b6d\",\n urls = [\n \"https://mirror.bazel.build/github.com/google/googletest/archive/e3bd4cbeaeef3cee65a68a8bd3c535cb779e9b6d.tar.gz\",\n \"https://github.com/google/googletest/archive/e3bd4cbeaeef3cee65a68a8bd3c535cb779e9b6d.tar.gz\",\n ],\n )\n\n _maybe(\n http_archive,\n name = \"com_github_grpc_grpc\",\n strip_prefix = \"grpc-1.16.1\",\n urls = [\n \"https://github.com/grpc/grpc/archive/v1.16.1.tar.gz\",\n ],\n )\n\ndef load_trtis():\n http_archive(\n name = \"com_github_nvidia_trtis\",\n strip_prefix = \"tensorrt-inference-server-0.9.0\",\n urls = [\n \"https://github.com/NVIDIA/tensorrt-inference-server/archive/v0.9.0.tar.gz\",\n ],\n )\n\ndef load_benchmark():\n http_archive(\n name = \"com_github_google_benchmark\",\n sha256 = \"f8e525db3c42efc9c7f3bc5176a8fa893a9a9920bbd08cef30fb56a51854d60d\",\n strip_prefix = \"benchmark-1.4.1\",\n urls = [\n \"https://github.com/google/benchmark/archive/v1.4.1.tar.gz\",\n ],\n )\n\ndef _maybe(repo_rule, name, **kwargs):\n if name not in native.existing_rules():\n repo_rule(name = name, **kwargs)\n" }, { "path": "bazel/tensorrt_configure.bzl", "content": "\"\"\"Build rule generator for locally installed TensorRT.\"\"\"\n\n# inspired from: https://github.com/google/nvidia_libs_test\n\ndef _get_env_var(repository_ctx, name, default):\n if name in repository_ctx.os.environ:\n return repository_ctx.os.environ[name]\n return default\n\ndef _impl(repository_ctx):\n hdrs_path = _get_env_var(repository_ctx, \"TENSORRT_HDRS_PATH\", \"/usr/include/x86_64-linux-gnu\")\n libs_path = _get_env_var(repository_ctx, \"TENSORRT_LIBS_PATH\", \"/usr/lib/x86_64-linux-gnu\")\n\n print(\"Using TensorRT Headers from %s\\n\" % hdrs_path)\n print(\"Using TensorRT Libs from %s\\n\" % libs_path)\n\n repository_ctx.symlink(hdrs_path, \"include\")\n repository_ctx.symlink(libs_path, \"libs\")\n\n repository_ctx.file(\"BUILD\", \"\"\"\npackage(default_visibility = [\"//visibility:public\"])\n\n# The *_headers cc_library rules below aren't cc_inc_library rules because\n# dependent targets would only see the first one.\n\ncc_library(\n name = \"tensorrt_headers\",\n hdrs = glob(\n include = [\"include/Nv*.h\"],\n ),\n strip_include_prefix = \"include\",\n # Allows including CUDA headers with angle brackets.\n # includes = [\"cuda/include\"],\n)\n\ncc_library(\n name = \"tensorrt_infer\",\n srcs = [\"libs/libnvinfer.so\"],\n linkopts = [\"-ldl\"],\n)\n\n\"\"\")\n\ntensorrt_configure = repository_rule(\n implementation = _impl,\n environ = [\"TENSORRT_HDRS_PATH\", \"TENSORRT_LIBS_PATH\"],\n)\n" }, { "path": "build.sh", "content": "#!/bin/bash\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\nmkdir -p build\ncd build\ncmake -DCMAKE_BUILD_TYPE=Release ..\ncmake -DCMAKE_BUILD_TYPE=Release ..\nmake -j\ncd /work/notebooks\nln -f -s /work/build/tensorrt-laboratory/python/trtlab/trtlab.cpython-35m-x86_64-linux-gnu.so\n#make install\n" }, { "path": "cmake/FindTensorRT.cmake", "content": "# This module defines the following variables:\n#\n# ::\n#\n# TensorRT_INCLUDE_DIRS\n# TensorRT_LIBRARIES\n# TensorRT_FOUND\n#\n# ::\n#\n# TensorRT_VERSION_STRING - version (x.y.z)\n# TensorRT_VERSION_MAJOR - major version (x)\n# TensorRT_VERSION_MINOR - minor version (y)\n# TensorRT_VERSION_PATCH - patch version (z)\n#\n# Hints\n# ^^^^^\n# A user may set ``TensorRT_ROOT`` to an installation root to tell this module where to look.\n#\nset(_TensorRT_SEARCHES)\n\nif(TensorRT_ROOT)\n set(_TensorRT_SEARCH_ROOT PATHS ${TensorRT_ROOT} NO_DEFAULT_PATH)\n list(APPEND _TensorRT_SEARCHES _TensorRT_SEARCH_ROOT)\nendif()\n\n# appends some common paths\nset(_TensorRT_SEARCH_NORMAL\n PATHS \"/usr\"\n)\nlist(APPEND _TensorRT_SEARCHES _TensorRT_SEARCH_NORMAL)\n\n# Include dir\nforeach(search ${_TensorRT_SEARCHES})\n find_path(TensorRT_INCLUDE_DIR NAMES NvInfer.h ${${search}} PATH_SUFFIXES include)\nendforeach()\n\nif(NOT TensorRT_LIBRARY)\n foreach(search ${_TensorRT_SEARCHES})\n find_library(TensorRT_LIBRARY NAMES nvinfer ${${search}} PATH_SUFFIXES lib)\n endforeach()\nendif()\n\nmark_as_advanced(TensorRT_INCLUDE_DIR)\n\nif(TensorRT_INCLUDE_DIR AND EXISTS \"${TensorRT_INCLUDE_DIR}/NvInfer.h\")\n file(STRINGS \"${TensorRT_INCLUDE_DIR}/NvInfer.h\" TensorRT_MAJOR REGEX \"^#define NV_TENSORRT_MAJOR [0-9]+.*$\")\n file(STRINGS \"${TensorRT_INCLUDE_DIR}/NvInfer.h\" TensorRT_MINOR REGEX \"^#define NV_TENSORRT_MINOR [0-9]+.*$\")\n file(STRINGS \"${TensorRT_INCLUDE_DIR}/NvInfer.h\" TensorRT_PATCH REGEX \"^#define NV_TENSORRT_PATCH [0-9]+.*$\")\n\n string(REGEX REPLACE \"^#define NV_TENSORRT_MAJOR ([0-9]+).*$\" \"\\\\1\" TensorRT_VERSION_MAJOR \"${TensorRT_MAJOR}\")\n string(REGEX REPLACE \"^#define NV_TENSORRT_MINOR ([0-9]+).*$\" \"\\\\1\" TensorRT_VERSION_MINOR \"${TensorRT_MINOR}\")\n string(REGEX REPLACE \"^#define NV_TENSORRT_PATCH ([0-9]+).*$\" \"\\\\1\" TensorRT_VERSION_PATCH \"${TensorRT_PATCH}\")\n set(TensorRT_VERSION_STRING \"${TensorRT_VERSION_MAJOR}.${TensorRT_VERSION_MINOR}.${TensorRT_VERSION_PATCH}\")\nendif()\n\ninclude(FindPackageHandleStandardArgs)\nFIND_PACKAGE_HANDLE_STANDARD_ARGS(TensorRT REQUIRED_VARS TensorRT_LIBRARY TensorRT_INCLUDE_DIR VERSION_VAR TensorRT_VERSION_STRING)\n\nif(TensorRT_FOUND)\n set(TensorRT_INCLUDE_DIRS ${TensorRT_INCLUDE_DIR})\n\n if(NOT TensorRT_LIBRARIES)\n set(TensorRT_LIBRARIES ${TensorRT_LIBRARY})\n endif()\n\n if(NOT TARGET TensorRT::TensorRT)\n add_library(TensorRT::TensorRT UNKNOWN IMPORTED)\n set_target_properties(TensorRT::TensorRT PROPERTIES INTERFACE_INCLUDE_DIRECTORIES \"${TensorRT_INCLUDE_DIRS}\")\n set_property(TARGET TensorRT::TensorRT APPEND PROPERTY IMPORTED_LOCATION \"${TensorRT_LIBRARY}\")\n endif()\nendif()\n" }, { "path": "cmake/Findcpuaff.cmake", "content": "# This module defines the following variables:\n#\n# ::\n#\n# CPUAFF_INCLUDE_DIRS\n# CPUAFF_FOUND\n#\n# ::\n#\n# Hints\n# ^^^^^\n# A user may set ``CPUAFF_ROOT`` to an installation root to tell this module where to look.\n#\nset(CPUAFF_FOUND FALSE)\nset(_CPUAFF_SEARCHES)\n\nif(CPUAFF_ROOT)\n set(_CPUAFF_SEARCH_ROOT PATHS ${CPUAFF_ROOT} NO_DEFAULT_PATH)\n list(APPEND _CPUAFF_SEARCHES _CPUAFF_SEARCH_ROOT)\nelse()\n list(APPEND _CPUAFF_SEARCHES \"/usr\")\n list(APPEND _CPUAFF_SEARCHES \"/usr/local\")\nendif()\n\n# Include dir\nforeach(search ${_CPUAFF_SEARCHES})\n find_path(\n CPUAFF_INCLUDE_DIR \n NAMES cpuaff/cpuaff.hpp\n PATHS ${CPUAFF_ROOT}\n PATH_SUFFIXES include)\n message(STATUS \"cpuaff: ${CPUAFF_INCLUDE_DIR}\")\nendforeach()\n\nmark_as_advanced(CPUAFF_INCLUDE_DIR)\n\nif(CPUAFF_INCLUDE_DIR AND EXISTS \"${CPUAFF_INCLUDE_DIR}/cpuaff/cpuaff.hpp\")\n set(CPUAFF_FOUND True)\n\n add_library(cpuaff INTERFACE)\n target_include_directories(cpuaff INTERFACE ${CPUAFF_INCLUDE_DIR})\nendif()\n" }, { "path": "cmake/GRPCGenerateCPP.cmake", "content": "\nfind_package(gRPC REQUIRED COMPONENTS grpc_cpp_plugin)\nset(_gRPC_CPP_PLUGIN_EXECUTABLE $)\n\nfunction(PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS)\n cmake_parse_arguments(protobuf \"\" \"EXPORT_MACRO;DESCRIPTORS\" \"\" ${ARGN})\n\n set(PROTO_FILES \"${protobuf_UNPARSED_ARGUMENTS}\")\n if(NOT PROTO_FILES)\n message(SEND_ERROR \"Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files\")\n return()\n endif()\n\n if(PROTOBUF_GENERATE_CPP_APPEND_PATH) # This variable is common for all types of output.\n # Create an include path for each file specified\n foreach(FIL ${PROTO_FILES})\n get_filename_component(ABS_FIL ${FIL} ABSOLUTE)\n get_filename_component(ABS_PATH ${ABS_FIL} PATH)\n list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)\n if(${_contains_already} EQUAL -1)\n list(APPEND _protobuf_include_path -I ${ABS_PATH})\n endif()\n endforeach()\n else()\n set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})\n endif()\n\n if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)\n set(Protobuf_IMPORT_DIR \"${PROTOBUF_IMPORT_DIRS\")\n endif()\n\n if(DEFINED Protobuf_IMPORT_DIRS)\n foreach(DIR ${Protobuf_IMPORT_DIRS})\n get_filename_component(ABS_PATH ${DIR} ABSOLUTE)\n list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)\n if(${_contains_already} EQUAL -1)\n list(APPEND _protobuf_include_path -I ${ABS_PATH})\n endif()\n endforeach()\n endif()\n\n set(${SRCS})\n set(${HDRS})\n foreach(FIL ${PROTO_FILES})\n get_filename_component(ABS_FIL ${FIL} ABSOLUTE)\n get_filename_component(FIL_WE ${FIL} NAME_WE)\n\n if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH)\n get_filename_component(FIL_DIR ${FIL} DIRECTORY)\n if(FIL_DIR)\n set(FIL_WE \"${FIL_DIR}/${FIL_WE}\")\n endif()\n endif()\n\n set(_protobuf_grpc_src \"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc\")\n set(_protobuf_grpc_hdr \"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h\")\n list(APPEND ${SRCS} \"${_protobuf_grpc_src}\")\n list(APPEND ${HDRS} \"${_protobuf_grpc_hdr}\")\n\n add_custom_command(\n OUTPUT \"${_protobuf_grpc_src}\"\n \"${_protobuf_grpc_hdr}\"\n COMMAND ${Protobuf_PROTOC_EXECUTABLE}\n --grpc_out=${CMAKE_CURRENT_BINARY_DIR}\n --plugin=protoc-gen-grpc=${_gRPC_CPP_PLUGIN_EXECUTABLE}\n ${_protobuf_include_path} ${ABS_FIL}\n DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE}\n COMMENT \"Running gRPC C++ protocol buffer compiler on ${FIL}\"\n VERBATIM)\n endforeach()\n\n set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)\n set(${SRCS} ${${SRCS}} PARENT_SCOPE)\n set(${HDRS} ${${HDRS}} PARENT_SCOPE)\nendfunction()" }, { "path": "cmake/GRPCGenerateCPPLikeBazel.cmake", "content": "find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin) # Get full path to plugin\n\nfunction(PROTOBUF_GENERATE_GRPC_CPP_LIKE_BAZEL SRCS HDRS)\n cmake_parse_arguments(protobuf \"\" \"EXPORT_MACRO;DESCRIPTORS\" \"\" ${ARGN})\n\n set(PROTO_FILES \"${protobuf_UNPARSED_ARGUMENTS}\")\n if(NOT PROTO_FILES)\n message(SEND_ERROR \"Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files\")\n return()\n endif()\n\n if(protobuf_EXPORT_MACRO)\n set(DLL_EXPORT_DECL \"dllexport_decl=${protobuf_EXPORT_MACRO}:\")\n endif()\n\n get_filename_component(ABS_PROTO_PATH ${CMAKE_SOURCE_DIR} ABSOLUTE)\n set(EXTRA_ARGS \"--proto_path=${ABS_PROTO_PATH}\")\n file(RELATIVE_PATH Protobuf_PRE_IMPORT_DIRS ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})\n\n if(PROTOBUF_GENERATE_CPP_APPEND_PATH) # This variable is common for all types of output.\n # Create an include path for each file specified\n foreach(FIL ${PROTO_FILES})\n get_filename_component(ABS_FIL ${FIL} ABSOLUTE)\n get_filename_component(ABS_PATH ${ABS_FIL} PATH)\n list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)\n if(${_contains_already} EQUAL -1)\n list(APPEND _protobuf_include_path -I ${ABS_PATH})\n endif()\n endforeach()\n else()\n set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})\n endif()\n\n if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)\n set(Protobuf_IMPORT_DIR \"${PROTOBUF_IMPORT_DIRS\")\n endif()\n\n if(DEFINED Protobuf_IMPORT_DIRS)\n foreach(DIR ${Protobuf_IMPORT_DIRS})\n get_filename_component(ABS_PATH ${DIR} ABSOLUTE)\n list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)\n if(${_contains_already} EQUAL -1)\n list(APPEND _protobuf_include_path -I ${ABS_PATH})\n endif()\n endforeach()\n endif()\n\n set(${SRCS})\n set(${HDRS})\n foreach(FIL ${PROTO_FILES})\n message(STATUS \"grpc_cpp_proto: ${FIL}\")\n get_filename_component(ABS_FIL ${FIL} ABSOLUTE)\n get_filename_component(FIL_WE ${FIL} NAME_WE)\n message(STATUS \"grpc_cpp_proto_abs: ${ABS_FIL}\")\n\n if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH)\n get_filename_component(FIL_DIR ${FIL} DIRECTORY)\n if(FIL_DIR)\n set(FIL_WE \"${FIL_DIR}/${FIL_WE}\")\n endif()\n endif()\n\n if(Protobuf_PRE_IMPORT_DIRS)\n set(_protobuf_protoc_src \"${CMAKE_CURRENT_BINARY_DIR}/${Protobuf_PRE_IMPORT_DIRS}/${FIL_WE}.grpc.pb.cc\")\n set(_protobuf_protoc_hdr \"${CMAKE_CURRENT_BINARY_DIR}/${Protobuf_PRE_IMPORT_DIRS}/${FIL_WE}.grpc.pb.h\")\n else()\n set(_protobuf_protoc_src \"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc\")\n set(_protobuf_protoc_hdr \"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h\")\n endif()\n message(STATUS \"grpc_cpp_src: ${_protobuf_protoc_src}\")\n list(APPEND ${SRCS} \"${_protobuf_protoc_src}\")\n list(APPEND ${HDRS} \"${_protobuf_protoc_hdr}\")\n\n add_custom_command(\n OUTPUT \"${_protobuf_protoc_src}\"\n \"${_protobuf_protoc_hdr}\"\n COMMAND ${Protobuf_PROTOC_EXECUTABLE}\n ${EXTRA_ARGS}\n \"--grpc_out=${CMAKE_CURRENT_BINARY_DIR}\"\n \"--plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN}\"\n ${_protobuf_protoc_flags}\n ${_protobuf_include_path} ${ABS_FIL}\n DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE}\n COMMENT \"Running gRPC C++ protocol buffer compiler on ${FIL}\"\n VERBATIM)\n endforeach()\n\n set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)\n set(${SRCS} ${${SRCS}} PARENT_SCOPE)\n set(${HDRS} ${${HDRS}} PARENT_SCOPE)\nendfunction()" }, { "path": "cmake/LibFindMacros.cmake", "content": "# Version 2.2\n# Public Domain, originally written by Lasse Kärkkäinen \n# Maintained at https://github.com/Tronic/cmake-modules\n# Please send your improvements as pull requests on Github.\n\n# Find another package and make it a dependency of the current package.\n# This also automatically forwards the \"REQUIRED\" argument.\n# Usage: libfind_package( [extra args to find_package])\nmacro (libfind_package PREFIX PKG)\n set(${PREFIX}_args ${PKG} ${ARGN})\n if (${PREFIX}_FIND_REQUIRED)\n set(${PREFIX}_args ${${PREFIX}_args} REQUIRED)\n endif()\n find_package(${${PREFIX}_args})\n set(${PREFIX}_DEPENDENCIES ${${PREFIX}_DEPENDENCIES};${PKG})\n unset(${PREFIX}_args)\nendmacro()\n\n# A simple wrapper to make pkg-config searches a bit easier.\n# Works the same as CMake's internal pkg_check_modules but is always quiet.\nmacro (libfind_pkg_check_modules)\n find_package(PkgConfig QUIET)\n if (PKG_CONFIG_FOUND)\n pkg_check_modules(${ARGN} QUIET)\n endif()\nendmacro()\n\n# Avoid useless copy&pasta by doing what most simple libraries do anyway:\n# pkg-config, find headers, find library.\n# Usage: libfind_pkg_detect( FIND_PATH [other args] FIND_LIBRARY [other args])\n# E.g. libfind_pkg_detect(SDL2 sdl2 FIND_PATH SDL.h PATH_SUFFIXES SDL2 FIND_LIBRARY SDL2)\nfunction (libfind_pkg_detect PREFIX)\n # Parse arguments\n set(argname pkgargs)\n foreach (i ${ARGN})\n if (\"${i}\" STREQUAL \"FIND_PATH\")\n set(argname pathargs)\n elseif (\"${i}\" STREQUAL \"FIND_LIBRARY\")\n set(argname libraryargs)\n else()\n set(${argname} ${${argname}} ${i})\n endif()\n endforeach()\n if (NOT pkgargs)\n message(FATAL_ERROR \"libfind_pkg_detect requires at least a pkg_config package name to be passed.\")\n endif()\n # Find library\n libfind_pkg_check_modules(${PREFIX}_PKGCONF ${pkgargs})\n if (pathargs)\n find_path(${PREFIX}_INCLUDE_DIR NAMES ${pathargs} HINTS ${${PREFIX}_PKGCONF_INCLUDE_DIRS})\n endif()\n if (libraryargs)\n find_library(${PREFIX}_LIBRARY NAMES ${libraryargs} HINTS ${${PREFIX}_PKGCONF_LIBRARY_DIRS})\n endif()\nendfunction()\n\n# Extracts a version #define from a version.h file, output stored to _VERSION.\n# Usage: libfind_version_header(Foobar foobar/version.h FOOBAR_VERSION_STR)\n# Fourth argument \"QUIET\" may be used for silently testing different define names.\n# This function does nothing if the version variable is already defined.\nfunction (libfind_version_header PREFIX VERSION_H DEFINE_NAME)\n # Skip processing if we already have a version or if the include dir was not found\n if (${PREFIX}_VERSION OR NOT ${PREFIX}_INCLUDE_DIR)\n return()\n endif()\n set(quiet ${${PREFIX}_FIND_QUIETLY})\n # Process optional arguments\n foreach(arg ${ARGN})\n if (arg STREQUAL \"QUIET\")\n set(quiet TRUE)\n else()\n message(AUTHOR_WARNING \"Unknown argument ${arg} to libfind_version_header ignored.\")\n endif()\n endforeach()\n # Read the header and parse for version number\n set(filename \"${${PREFIX}_INCLUDE_DIR}/${VERSION_H}\")\n if (NOT EXISTS ${filename})\n if (NOT quiet)\n message(AUTHOR_WARNING \"Unable to find ${${PREFIX}_INCLUDE_DIR}/${VERSION_H}\")\n endif()\n return()\n endif()\n file(READ \"${filename}\" header)\n string(REGEX REPLACE \".*#[ \\t]*define[ \\t]*${DEFINE_NAME}[ \\t]*\\\"([^\\n]*)\\\".*\" \"\\\\1\" match \"${header}\")\n # No regex match?\n if (match STREQUAL header)\n if (NOT quiet)\n message(AUTHOR_WARNING \"Unable to find \\#define ${DEFINE_NAME} \\\"\\\" from ${${PREFIX}_INCLUDE_DIR}/${VERSION_H}\")\n endif()\n return()\n endif()\n # Export the version string\n set(${PREFIX}_VERSION \"${match}\" PARENT_SCOPE)\nendfunction()\n\n# Do the final processing once the paths have been detected.\n# If include dirs are needed, ${PREFIX}_PROCESS_INCLUDES should be set to contain\n# all the variables, each of which contain one include directory.\n# Ditto for ${PREFIX}_PROCESS_LIBS and library files.\n# Will set ${PREFIX}_FOUND, ${PREFIX}_INCLUDE_DIRS and ${PREFIX}_LIBRARIES.\n# Also handles errors in case library detection was required, etc.\nfunction (libfind_process PREFIX)\n # Skip processing if already processed during this configuration run\n if (${PREFIX}_FOUND)\n return()\n endif()\n\n set(found TRUE) # Start with the assumption that the package was found\n\n # Did we find any files? Did we miss includes? These are for formatting better error messages.\n set(some_files FALSE)\n set(missing_headers FALSE)\n\n # Shorthands for some variables that we need often\n set(quiet ${${PREFIX}_FIND_QUIETLY})\n set(required ${${PREFIX}_FIND_REQUIRED})\n set(exactver ${${PREFIX}_FIND_VERSION_EXACT})\n set(findver \"${${PREFIX}_FIND_VERSION}\")\n set(version \"${${PREFIX}_VERSION}\")\n\n # Lists of config option names (all, includes, libs)\n unset(configopts)\n set(includeopts ${${PREFIX}_PROCESS_INCLUDES})\n set(libraryopts ${${PREFIX}_PROCESS_LIBS})\n\n # Process deps to add to \n foreach (i ${PREFIX} ${${PREFIX}_DEPENDENCIES})\n if (DEFINED ${i}_INCLUDE_OPTS OR DEFINED ${i}_LIBRARY_OPTS)\n # The package seems to export option lists that we can use, woohoo!\n list(APPEND includeopts ${${i}_INCLUDE_OPTS})\n list(APPEND libraryopts ${${i}_LIBRARY_OPTS})\n else()\n # If plural forms don't exist or they equal singular forms\n if ((NOT DEFINED ${i}_INCLUDE_DIRS AND NOT DEFINED ${i}_LIBRARIES) OR\n ({i}_INCLUDE_DIR STREQUAL ${i}_INCLUDE_DIRS AND ${i}_LIBRARY STREQUAL ${i}_LIBRARIES))\n # Singular forms can be used\n if (DEFINED ${i}_INCLUDE_DIR)\n list(APPEND includeopts ${i}_INCLUDE_DIR)\n endif()\n if (DEFINED ${i}_LIBRARY)\n list(APPEND libraryopts ${i}_LIBRARY)\n endif()\n else()\n # Oh no, we don't know the option names\n message(FATAL_ERROR \"We couldn't determine config variable names for ${i} includes and libs. Aieeh!\")\n endif()\n endif()\n endforeach()\n \n if (includeopts)\n list(REMOVE_DUPLICATES includeopts)\n endif()\n \n if (libraryopts)\n list(REMOVE_DUPLICATES libraryopts)\n endif()\n\n string(REGEX REPLACE \".*[ ;]([^ ;]*(_INCLUDE_DIRS|_LIBRARIES))\" \"\\\\1\" tmp \"${includeopts} ${libraryopts}\")\n if (NOT tmp STREQUAL \"${includeopts} ${libraryopts}\")\n message(AUTHOR_WARNING \"Plural form ${tmp} found in config options of ${PREFIX}. This works as before but is now deprecated. Please only use singular forms INCLUDE_DIR and LIBRARY, and update your find scripts for LibFindMacros > 2.0 automatic dependency system (most often you can simply remove the PROCESS variables entirely).\")\n endif()\n\n # Include/library names separated by spaces (notice: not CMake lists)\n unset(includes)\n unset(libs)\n\n # Process all includes and set found false if any are missing\n foreach (i ${includeopts})\n list(APPEND configopts ${i})\n if (NOT \"${${i}}\" STREQUAL \"${i}-NOTFOUND\")\n list(APPEND includes \"${${i}}\")\n else()\n set(found FALSE)\n set(missing_headers TRUE)\n endif()\n endforeach()\n\n # Process all libraries and set found false if any are missing\n foreach (i ${libraryopts})\n list(APPEND configopts ${i})\n if (NOT \"${${i}}\" STREQUAL \"${i}-NOTFOUND\")\n list(APPEND libs \"${${i}}\")\n else()\n set (found FALSE)\n endif()\n endforeach()\n\n # Version checks\n if (found AND findver)\n if (NOT version)\n message(WARNING \"The find module for ${PREFIX} does not provide version information, so we'll just assume that it is OK. Please fix the module or remove package version requirements to get rid of this warning.\")\n elseif (version VERSION_LESS findver OR (exactver AND NOT version VERSION_EQUAL findver))\n set(found FALSE)\n set(version_unsuitable TRUE)\n endif()\n endif()\n\n # If all-OK, hide all config options, export variables, print status and exit\n if (found)\n foreach (i ${configopts})\n mark_as_advanced(${i})\n endforeach()\n if (NOT quiet)\n message(STATUS \"Found ${PREFIX} ${${PREFIX}_VERSION}\")\n if (LIBFIND_DEBUG)\n message(STATUS \" ${PREFIX}_DEPENDENCIES=${${PREFIX}_DEPENDENCIES}\")\n message(STATUS \" ${PREFIX}_INCLUDE_OPTS=${includeopts}\")\n message(STATUS \" ${PREFIX}_INCLUDE_DIRS=${includes}\")\n message(STATUS \" ${PREFIX}_LIBRARY_OPTS=${libraryopts}\")\n message(STATUS \" ${PREFIX}_LIBRARIES=${libs}\")\n endif()\n set (${PREFIX}_INCLUDE_OPTS ${includeopts} PARENT_SCOPE)\n set (${PREFIX}_LIBRARY_OPTS ${libraryopts} PARENT_SCOPE)\n set (${PREFIX}_INCLUDE_DIRS ${includes} PARENT_SCOPE)\n set (${PREFIX}_LIBRARIES ${libs} PARENT_SCOPE)\n set (${PREFIX}_FOUND TRUE PARENT_SCOPE)\n endif()\n return() \n endif()\n\n # Format messages for debug info and the type of error\n set(vars \"Relevant CMake configuration variables:\\n\")\n foreach (i ${configopts})\n mark_as_advanced(CLEAR ${i})\n set(val ${${i}})\n if (\"${val}\" STREQUAL \"${i}-NOTFOUND\")\n set (val \"\")\n elseif (val AND NOT EXISTS ${val})\n set (val \"${val} (does not exist)\")\n else()\n set(some_files TRUE)\n endif()\n set(vars \"${vars} ${i}=${val}\\n\")\n endforeach()\n set(vars \"${vars}You may use CMake GUI, cmake -D or ccmake to modify the values. Delete CMakeCache.txt to discard all values and force full re-detection if necessary.\\n\")\n if (version_unsuitable)\n set(msg \"${PREFIX} ${${PREFIX}_VERSION} was found but\")\n if (exactver)\n set(msg \"${msg} only version ${findver} is acceptable.\")\n else()\n set(msg \"${msg} version ${findver} is the minimum requirement.\")\n endif()\n else()\n if (missing_headers)\n set(msg \"We could not find development headers for ${PREFIX}. Do you have the necessary dev package installed?\")\n elseif (some_files)\n set(msg \"We only found some files of ${PREFIX}, not all of them. Perhaps your installation is incomplete or maybe we just didn't look in the right place?\")\n if(findver)\n set(msg \"${msg} This could also be caused by incompatible version (if it helps, at least ${PREFIX} ${findver} should work).\")\n endif()\n else()\n set(msg \"We were unable to find package ${PREFIX}.\")\n endif()\n endif()\n\n # Fatal error out if REQUIRED\n if (required)\n set(msg \"REQUIRED PACKAGE NOT FOUND\\n${msg} This package is REQUIRED and you need to install it or adjust CMake configuration in order to continue building ${CMAKE_PROJECT_NAME}.\")\n message(FATAL_ERROR \"${msg}\\n${vars}\")\n endif()\n # Otherwise just print a nasty warning\n if (NOT quiet)\n message(WARNING \"WARNING: MISSING PACKAGE\\n${msg} This package is NOT REQUIRED and you may ignore this warning but by doing so you may miss some functionality of ${CMAKE_PROJECT_NAME}. \\n${vars}\")\n endif()\nendfunction()\n\n" }, { "path": "cmake/ProtobufGenerateCPPLikeBazel.cmake", "content": "function(PROTOBUF_GENERATE_CPP_LIKE_BAZEL SRCS HDRS)\n cmake_parse_arguments(protobuf \"\" \"EXPORT_MACRO;DESCRIPTORS\" \"\" ${ARGN})\n\n set(PROTO_FILES \"${protobuf_UNPARSED_ARGUMENTS}\")\n if(NOT PROTO_FILES)\n message(SEND_ERROR \"Error: PROTOBUF_GENERATE_CPP() called without any proto files\")\n return()\n endif()\n\n if(protobuf_EXPORT_MACRO)\n set(DLL_EXPORT_DECL \"dllexport_decl=${protobuf_EXPORT_MACRO}:\")\n endif()\n\n get_filename_component(ABS_PROTO_PATH ${CMAKE_SOURCE_DIR} ABSOLUTE)\n set(EXTRA_ARGS \"--proto_path=${ABS_PROTO_PATH}\")\n file(RELATIVE_PATH Protobuf_PRE_IMPORT_DIRS ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})\n\n if(PROTOBUF_GENERATE_CPP_APPEND_PATH)\n # Create an include path for each file specified\n foreach(FIL ${PROTO_FILES})\n get_filename_component(ABS_FIL ${FIL} ABSOLUTE)\n get_filename_component(ABS_PATH ${ABS_FIL} PATH)\n list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)\n if(${_contains_already} EQUAL -1)\n list(APPEND _protobuf_include_path -I ${ABS_PATH})\n endif()\n endforeach()\n else()\n set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})\n endif()\n\n if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)\n set(Protobuf_IMPORT_DIRS \"${PROTOBUF_IMPORT_DIRS}\")\n endif()\n\n if(DEFINED Protobuf_IMPORT_DIRS)\n foreach(DIR ${Protobuf_IMPORT_DIRS})\n get_filename_component(ABS_PATH ${DIR} ABSOLUTE)\n list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)\n if(${_contains_already} EQUAL -1)\n list(APPEND _protobuf_include_path -I ${ABS_PATH})\n endif()\n endforeach()\n endif()\n\n set(${SRCS})\n set(${HDRS})\n if (protobuf_DESCRIPTORS)\n set(${protobuf_DESCRIPTORS})\n endif()\n\n foreach(FIL ${PROTO_FILES})\n get_filename_component(ABS_FIL ${FIL} ABSOLUTE)\n get_filename_component(FIL_WE ${FIL} NAME_WE)\n if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH)\n get_filename_component(FIL_DIR ${FIL} DIRECTORY)\n if(FIL_DIR)\n set(FIL_WE \"${FIL_DIR}/${FIL_WE}\")\n endif()\n endif()\n\n if(Protobuf_PRE_IMPORT_DIRS)\n set(_protobuf_protoc_src \"${CMAKE_CURRENT_BINARY_DIR}/${Protobuf_PRE_IMPORT_DIRS}/${FIL_WE}.pb.cc\")\n set(_protobuf_protoc_hdr \"${CMAKE_CURRENT_BINARY_DIR}/${Protobuf_PRE_IMPORT_DIRS}/${FIL_WE}.pb.h\")\n else()\n set(_protobuf_protoc_src \"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc\")\n set(_protobuf_protoc_hdr \"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h\")\n endif()\n list(APPEND ${SRCS} \"${_protobuf_protoc_src}\")\n list(APPEND ${HDRS} \"${_protobuf_protoc_hdr}\")\n\n if(protobuf_DESCRIPTORS)\n set(_protobuf_protoc_desc \"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.desc\")\n set(_protobuf_protoc_flags \"--descriptor_set_out=${_protobuf_protoc_desc}\")\n list(APPEND ${protobuf_DESCRIPTORS} \"${_protobuf_protoc_desc}\")\n else()\n set(_protobuf_protoc_desc \"\")\n set(_protobuf_protoc_flags \"\")\n endif()\n\n add_custom_command(\n OUTPUT \"${_protobuf_protoc_src}\"\n \"${_protobuf_protoc_hdr}\"\n ${_protobuf_protoc_desc}\n COMMAND protobuf::protoc\n ${EXTRA_ARGS}\n \"--cpp_out=${DLL_EXPORT_DECL}${CMAKE_CURRENT_BINARY_DIR}\"\n ${_protobuf_protoc_flags}\n ${_protobuf_include_path} ${ABS_FIL}\n DEPENDS ${ABS_FIL} protobuf::protoc\n COMMENT \"Running C++ protocol buffer compiler on ${FIL}\"\n VERBATIM )\n endforeach()\n\n set(${SRCS} \"${${SRCS}}\" PARENT_SCOPE)\n set(${HDRS} \"${${HDRS}}\" PARENT_SCOPE)\n if(protobuf_DESCRIPTORS)\n set(${protobuf_DESCRIPTORS} \"${${protobuf_DESCRIPTORS}}\" PARENT_SCOPE)\n endif()\nendfunction()\n" }, { "path": "cmake/dependencies.cmake", "content": "include (ExternalProject)\n\nset (DEPENDENCIES)\nset (EXTRA_CMAKE_ARGS)\n\n# trtlab external dependencies\nlist (APPEND DEPENDENCIES boost dlpack gflags glog benchmark googletest cpuaff jemalloc)\nlist (APPEND DEPENDENCIES grpc-repo protobuf c-ares grpc cub cnpy)\n\n# note on ubuntu 18.04, you need\n# apt install libz-dev libssl-dev\n\n# customize the folder for external projects\n# download, source and builds for dependencies \n# will be in /Dependencies\nset_property (DIRECTORY PROPERTY EP_BASE Dependencies)\n\n# all dependencies will be installed here\n# typical directories: bin, include and lib\nset (BUILD_ROOT ${CMAKE_CURRENT_BINARY_DIR}/Dependencies/Build)\nset (SOURCE_ROOT ${CMAKE_CURRENT_BINARY_DIR}/Dependencies/Source)\nset (INSTALL_ROOT ${CMAKE_CURRENT_BINARY_DIR}/local)\n\n# set cmake search paths to pick up installed .cmake files\nlist(INSERT CMAKE_MODULE_PATH 0 \"${INSTALL_ROOT}/lib/cmake\")\nlist(INSERT CMAKE_PREFIX_PATH 0 \"${INSTALL_ROOT}/lib/cmake\")\n\n# cmake config args forwarded to trtlab\nlist(APPEND EXTRA_CMAKE_ARGS\n -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}\n -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}\n# -DBoost_VERBOSE=ON\n -DBoost_USE_STATIC_LIBS=ON\n -DCPUAFF_ROOT=${INSTALL_ROOT}\n -DJEMALLOC_STATIC_LIBRARIES=${INSTALL_ROOT}/lib/libjemalloc_pic.a\n -DCUB_INCLUDE_DIR=${SOURCE_ROOT}/cub\n -DINSTALL_ROOT=${INSTALL_ROOT}\n)\n\n# short-cut to dependencies build path\nset (BUILD_ROOT ${CMAKE_CURRENT_BINARY_DIR}/Dependencies/Build)\n\n# Boost\n# =====\n# - Use static linking to avoid issues with system-wide installations of Boost.\n# - Use numa=on to ensure the numa component of fiber gets built\nset(BOOST_COMPONENTS \"context,fiber,filesystem\")\nExternalProject_Add (boost\n URL https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.gz\n URL_HASH SHA256=c66e88d5786f2ca4dbebb14e06b566fb642a1a6947ad8cc9091f9f445134143f\n CONFIGURE_COMMAND ./bootstrap.sh --prefix=${INSTALL_ROOT} --with-libraries=${BOOST_COMPONENTS} numa=on\n BUILD_COMMAND ./b2 link=static cxxflags=-fPIC cflags=-fPIC cxxflags=\"-std=c++14\" numa=on \n --build-dir=${BUILD_ROOT}/boost --stagedir=${BUILD_ROOT}/boost\n BUILD_IN_SOURCE 1\n INSTALL_COMMAND ./b2 install numa=on\n)\n\n# DLPack\n# ======\nExternalProject_Add(dlpack\n GIT_REPOSITORY \"https://github.com/dmlc/dlpack.git\"\n GIT_TAG \"master\"\n CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}\n)\n\n\n# gflags\n# ======\n# config, build and install to INSTALL_ROOT\nExternalProject_Add(gflags\n GIT_REPOSITORY \"https://github.com/gflags/gflags.git\"\n GIT_TAG \"v2.2.2\"\n CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}\n -DBUILD_SHARED_LIBS=ON\n -DBUILD_STATIC_LIBS=ON\n -DBUILD_PACKAGING=OFF\n -DBUILD_TESTING=OFF\n -DBUILD_CONFIG_TESTS=OFF\n -DINSTALL_HEADERS=ON\n -DBUILD_gflags_LIB=OFF\n -DBUILD_gflags_nothreads_LIB=ON\n -DGFLAGS_NAMESPACE=google\n)\n\n# glog\n# ====\n# - link against shared \n# - todo: compile with -DWITH_GFLAGS=OFF and remove gflags dependency\nExternalProject_Add(glog\n DEPENDS gflags\n GIT_REPOSITORY \"https://github.com/google/glog\"\n GIT_TAG \"v0.4.0\"\n CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}\n -DCMAKE_BUILD_TYPE=Release\n -DBUILD_TESTING=OFF\n)\n\n# google benchmark\n# ================\nExternalProject_Add(benchmark\n DEPENDS \n GIT_REPOSITORY https://github.com/google/benchmark.git\n GIT_TAG \"v1.5.0\"\n BINARY_DIR \"${CMAKE_CURRENT_BINARY_DIR}/Dependencies/Build/benchmark\"\n CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}\n -DCMAKE_BUILD_TYPE=Release\n -DBENCHMARK_ENABLE_TESTING=OFF\n)\n\n# google test\n# ===========\nExternalProject_Add(googletest\n DEPENDS glog gflags\n GIT_REPOSITORY https://github.com/google/googletest.git\n GIT_TAG \"release-1.10.0\"\n BINARY_DIR \"${CMAKE_CURRENT_BINARY_DIR}/Dependencies/Build/googletest\"\n CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}\n -DCMAKE_BUILD_TYPE=Release\n)\n\n# cpuaff\n# ======\nExternalProject_Add(cpuaff\n URL http://dcdillon.github.io/cpuaff/releases/cpuaff-1.0.6.tar.gz\n CONFIGURE_COMMAND ./configure --prefix=${INSTALL_ROOT}\n BUILD_COMMAND make include\n INSTALL_COMMAND make install include\n BUILD_IN_SOURCE 1\n)\n\n# nvidia cub\n# ==========\nExternalProject_Add(cub\n GIT_REPOSITORY https://github.com/NVlabs/cub.git \n GIT_TAG \"1.8.0\"\n CONFIGURE_COMMAND \"\"\n BUILD_COMMAND \"\"\n INSTALL_COMMAND \"\"\n TEST_COMMAND \"\"\n)\n\n# jemalloc\n# ========\nExternalProject_Add(jemalloc\n URL https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2\n CONFIGURE_COMMAND ./configure --prefix=${INSTALL_ROOT}\n BUILD_COMMAND make include\n INSTALL_COMMAND make install include\n BUILD_IN_SOURCE 1\n)\n\n# cnpy - c++ library for reading and writing .npy/.npz files\n# ==========================================================\nExternalProject_Add(cnpy\n GIT_REPOSITORY \"https://github.com/rogersce/cnpy.git\"\n GIT_TAG \"master\"\n CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}\n -DCMAKE_BUILD_TYPE=Release\n -DBUILD_TESTING=OFF\n -DCMAKE_POSITION_INDEPENDENT_CODEL=ON\n)\n\n# grpc-repo\n# =========\nExternalProject_Add(grpc-repo\n GIT_REPOSITORY \"https://github.com/grpc/grpc.git\"\n GIT_TAG \"v1.32.0\"\n GIT_SUBMODULES \"third_party/cares/cares\" \"third_party/protobuf\" \"third_party/abseil-cpp\" \"third_party/re2\"\n CONFIGURE_COMMAND \"\"\n BUILD_COMMAND \"\"\n INSTALL_COMMAND \"\"\n TEST_COMMAND \"\"\n)\n\n#\n# Build protobuf project from grpc-repo\n#\nExternalProject_Add(absl\n SOURCE_DIR \"${SOURCE_ROOT}/grpc-repo/third_party/abseil-cpp\"\n DOWNLOAD_COMMAND \"\"\n CMAKE_CACHE_ARGS\n -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE\n -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_ROOT}\n DEPENDS grpc-repo\n)\n\nExternalProject_Add(re2\n SOURCE_DIR \"${SOURCE_ROOT}/grpc-repo/third_party/re2\"\n DOWNLOAD_COMMAND \"\"\n CMAKE_CACHE_ARGS\n -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE\n -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_ROOT}\n DEPENDS grpc-repo\n)\n\nExternalProject_Add(protobuf\n SOURCE_DIR \"${SOURCE_ROOT}/grpc-repo/third_party/protobuf/cmake\"\n DOWNLOAD_COMMAND \"\"\n CMAKE_ARGS\n -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON\n -Dprotobuf_BUILD_TESTS:BOOL=OFF\n -Dprotobuf_WITH_ZLIB:BOOL=OFF\n -Dprotobuf_MSVC_STATIC_RUNTIME:BOOL=OFF\n -DCMAKE_BUILD_TYPE=Release\n -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_ROOT}\n DEPENDS grpc-repo\n)\n\n# Location where protobuf-config.cmake will be installed varies by\n# platform\nif (WIN32)\n set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR \"${CMAKE_CURRENT_BINARY_DIR}/protobuf/cmake\")\nelse()\n set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR \"${INSTALL_ROOT}/lib/cmake\")\nendif()\n\n#\n# Build c-area project from grpc-repo\n#\nExternalProject_Add(c-ares\n SOURCE_DIR \"${SOURCE_ROOT}/grpc-repo/third_party/cares/cares\"\n DOWNLOAD_COMMAND \"\"\n CMAKE_ARGS\n -DCARES_SHARED:BOOL=OFF\n -DCARES_STATIC:BOOL=ON\n -DCARES_STATIC_PIC:BOOL=ON\n -DCMAKE_BUILD_TYPE=Release\n -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_ROOT}\n DEPENDS grpc-repo\n)\n\n\n#\n# Build GRPC\n#\nExternalProject_Add(grpc\n SOURCE_DIR \"${SOURCE_ROOT}/grpc-repo\"\n DOWNLOAD_COMMAND \"\"\n CMAKE_ARGS\n -DgRPC_INSTALL:BOOL=ON\n -DgRPC_BUILD_TESTS:BOOL=OFF\n -DgRPC_PROTOBUF_PROVIDER:STRING=package\n -DgRPC_PROTOBUF_PACKAGE_TYPE:STRING=CONFIG\n -DProtobuf_DIR:PATH=${INSTALL_ROOT}/lib/cmake\n -DgRPC_ZLIB_PROVIDER:STRING=package\n -DgRPC_CARES_PROVIDER:STRING=package\n -Dc-ares_DIR:PATH=${INSTALL_ROOT}/lib/cmake\n -DgRPC_SSL_PROVIDER:STRING=package\n -DgRPC_GFLAGS_PROVIDER=package\n -DgRPC_BENCHMARK_PROVIDER=package\n -DgRPC_RE2_PROVIDER:STRING=package\n -Dre2_DIR:STRING=${INSTALL_ROOT}/lib/cmake\n -DgRPC_ABSL_PROVIDER:STRING=package\n -Dabsl_DIR:STRING=${INSTALL_ROOT}/lib/cmake\n ${_CMAKE_ARGS_OPENSSL_ROOT_DIR}\n -DCMAKE_BUILD_TYPE=Release\n -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_ROOT}\n DEPENDS grpc-repo c-ares protobuf re2 absl gflags benchmark\n)\n\n\n\n# trtlab\n# ======\nExternalProject_Add (trtlab\n DEPENDS ${DEPENDENCIES}\n SOURCE_DIR ${PROJECT_SOURCE_DIR}\n CMAKE_ARGS -DBUILD_DEPENDENCIES=OFF ${EXTRA_CMAKE_ARGS}\n INSTALL_COMMAND \"\"\n BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})" }, { "path": "devel.sh", "content": "#!/bin/bash\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\nmodels_path=${TRT_MODELS_PATH:-\"/path/to/my/models\"}\nmodels_cli=\"\"\nif [ -d \"$models_path\" ]; then\n models_cli=\" -v $(realpath $models_path):/work/models \"\nfi\n\ncrt=\"\"\nif [ -x \"$(which luda)\" ] ; then\n echo \"Using luda\"\n crt=\"$(which luda) --no-home\"\nelif [ -x \"$(which nvidia-docker)\" ]; then\n echo \"Using nvidia-docker\"\n crt=\"nvidia-docker run --rm -ti\"\nelse\n echo \"No GPU container runtime found\"\n exit 911\nfi\nNV_GPU=0 $crt -v $PWD:/work $models_cli --workdir /work --name trtlab --net host trtlab\n" }, { "path": "examples/00_TensorRT/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\ninclude_directories(${CUDA_INCLUDE_DIRS})\ninclude_directories(${TensorRT_INCLUDE_DIRS})\n\n\n\nadd_executable(inference.x\n inference.cc\n ${PROTO_SRCS}\n ${PROTO_GRPC_SRCS})\n\ntarget_link_libraries(inference.x\n trtlab::nvrpc\n trtlab::tensorrt\n gflags\n)\n\nadd_executable(infer.x\n infer.cc\n ${PROTO_SRCS}\n ${PROTO_GRPC_SRCS})\n\ntarget_link_libraries(infer.x\n trtlab::nvrpc\n trtlab::tensorrt\n gflags\n)\n\nif(YAIS_ENABLE_MPI)\nfind_package(MPI)\ninclude_directories(SYSTEM ${MPI_INCLUDE_PATH})\ntarget_link_libraries(inference.x\n ${MPI_C_LIBRARIES}\n ${MPI_CXX_LIBRARIES}\n)\ntarget_compile_definitions(inference.x PUBLIC PLAYGROUND_USE_MPI)\nendif()\n" }, { "path": "examples/00_TensorRT/README.md", "content": "# Inference Example\n\nBasic CLI tool for executing TensorRT engines.\n\nProvide an engine and `inference.x` will run a simplifed inference pipeline using synthetic data.\n\nThe program will run a pipelined H2D -> TensorRT -> D2H calculation for `--seconds` (default: 5) with a\n0.1 second warmup run. By default, only 1 TensorRT Execution Context is used to perform the evaulation.\nYou can modify the number of contexts using the `--contexts`. Unless provided, the number of Input/Output\nBuffers is set to `(2 * contexts)`. See below for the list of [options](#options).\n\nThe `inference.x` program is fully pipelined and asynchronous. It performs uses three threads (default)\nto: 1) async copy input H2D, 2) launch the async inference evaluation and return output tensor to the host,\nand 3) to wait on the resouces used during execution and release them when finished. This final thread is\nwhere one might build a return message or do something else with the results.\n\nWhile running `inference.x`, you may find it useful to monitor GPU metrics using:\n```\nnvidia-smi dmon -i 0 -s put\n```\n\nNote: If you see numbers that differ from the output of `giexec`, you may have an IO bottleneck in that \nthe transfers are more expensive than the compute.\n\n * TODO: Update the program to output avg xfer time.\n * TODO: Build .engine files as part of the build\n\n## Quickstart\n\n```\nroot@dgx:/work/build/examples/00_TensorRT# ./inference.x --engine=/work/models/ResNet-50-b1-int8.engine\nI0702 22:16:51.868419 10857 TensorRT.cc:561] -- Initialzing TensorRT Resource Manager --\nI0702 22:16:51.868676 10857 TensorRT.cc:562] Maximum Execution Concurrency: 1\nI0702 22:16:51.868686 10857 TensorRT.cc:563] Maximum Copy Concurrency: 2\nI0702 22:16:53.430330 10857 TensorRT.cc:628] -- Registering Model: 0 --\nI0702 22:16:53.430399 10857 TensorRT.cc:629] Input/Output Tensors require 591.9 KiB\nI0702 22:16:53.430415 10857 TensorRT.cc:630] Execution Activations require 2.5 MiB\nI0702 22:16:53.430428 10857 TensorRT.cc:633] Weights require 30.7 MiB\nI0702 22:16:53.437571 10857 TensorRT.cc:652] -- Allocating TensorRT Resources --\nI0702 22:16:53.437587 10857 TensorRT.cc:653] Creating 1 TensorRT execution tokens.\nI0702 22:16:53.437595 10857 TensorRT.cc:654] Creating a Pool of 2 Host/Device Memory Stacks\nI0702 22:16:53.437607 10857 TensorRT.cc:655] Each Host Stack contains 608.0 KiB\nI0702 22:16:53.437614 10857 TensorRT.cc:656] Each Device Stack contains 3.2 MiB\nI0702 22:16:53.437623 10857 TensorRT.cc:657] Total GPU Memory: 6.5 MiB\nI0702 22:16:53.540400 10857 inference.cc:93] -- Inference: Running for ~5 seconds with batch_size 1 --\nI0702 22:16:58.543475 10857 inference.cc:131] Inference Results: 4770 batches in 5.00307 seconds; sec/batch: 0.00104886; inf/sec: 953.414\n```\n\n## Options\n```\n -buffers (Number of Buffers (default: 2x contexts)) type: int32 default: 0\n -contexts (Number of Execution Contexts) type: int32 default: 1\n -cudathreads (Number Cuda Launcher Threads) type: int32 default: 1\n -engine (TensorRT serialized engine) type: string\n default: \"/work/models/trt4.engine\"\n -respthreads (Number Response Sync Threads) type: int32 default: 1\n -seconds (Number of Execution Contexts) type: int32 default: 5\n```\n\n\n" }, { "path": "examples/00_TensorRT/infer.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n\n#include \n#include \n\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n#include \"tensorrt/laboratory/infer_bench.h\"\n#include \"tensorrt/laboratory/inference_manager.h\"\n#include \"tensorrt/laboratory/model.h\"\n#include \"tensorrt/laboratory/runtime.h\"\n\n#ifdef PLAYGROUND_USE_MPI\n#include \"mpi.h\"\n#define MPI_CHECK(mpicall) mpicall\n#else\n#define MPI_CHECK(mpicall)\n#endif\n\nusing trtlab::ThreadPool;\nusing trtlab::TensorRT::InferBench;\nusing trtlab::TensorRT::InferBenchKey;\nusing trtlab::TensorRT::InferenceManager;\nusing trtlab::TensorRT::ManagedRuntime;\nusing trtlab::TensorRT::Model;\nusing trtlab::TensorRT::Runtime;\nusing trtlab::TensorRT::StandardRuntime;\n\nstatic std::string ModelName(int model_id)\n{\n std::ostringstream stream;\n stream << model_id;\n return stream.str();\n}\n\nstatic bool ValidateEngine(const char* flagname, const std::string& value)\n{\n struct stat buffer;\n return (stat(value.c_str(), &buffer) == 0);\n}\n\nDEFINE_string(engine, \"/path/to/tensorrt.engine\", \"TensorRT serialized engine\");\nDEFINE_validator(engine, &ValidateEngine);\nDEFINE_string(runtime, \"default\", \"TensorRT Runtime\");\nDEFINE_int32(seconds, 5, \"Approximate number of seconds for the timing loop\");\nDEFINE_int32(contexts, 1, \"Number of Execution Contexts\");\nDEFINE_int32(buffers, 0, \"Number of Buffers (default: 2x contexts)\");\nDEFINE_int32(cudathreads, 1, \"Number Cuda Launcher Threads\");\nDEFINE_int32(respthreads, 1, \"Number Response Sync Threads\");\nDEFINE_int32(replicas, 1, \"Number of Replicas of the Model to load\");\nDEFINE_int32(batch_size, 0, \"Overrides the max batch_size of the provided engine\");\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"TensorRT Inference\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n MPI_CHECK(MPI_Init(&argc, &argv));\n\n auto contexts = FLAGS_contexts;\n auto buffers = FLAGS_buffers ? FLAGS_buffers : 2 * FLAGS_contexts;\n\n auto resources = std::make_shared(contexts, buffers);\n resources->RegisterThreadPool(\"pre\", std::make_unique(1));\n resources->RegisterThreadPool(\"cuda\", std::make_unique(1));\n resources->RegisterThreadPool(\"post\", std::make_unique(3));\n\n //, FLAGS_cudathreads, FLAGS_respthreads);\n std::shared_ptr runtime;\n if(FLAGS_runtime == \"default\")\n {\n runtime = std::make_shared();\n }\n else if(FLAGS_runtime == \"unified\")\n {\n runtime = std::make_shared();\n }\n else\n {\n LOG(FATAL) << \"Invalid TensorRT Runtime\";\n }\n\n std::vector> models;\n\n models.push_back(runtime->DeserializeEngine(FLAGS_engine));\n resources->RegisterModel(\"0\", models.back());\n resources->AllocateResources();\n\n auto batch_size = FLAGS_batch_size ? FLAGS_batch_size : models.back()->GetMaxBatchSize();\n\n for(int i = 1; i < FLAGS_replicas; i++)\n {\n models.push_back(runtime->DeserializeEngine(FLAGS_engine));\n resources->RegisterModel(ModelName(i), models.back());\n }\n\n {\n InferBench benchmark(resources);\n benchmark.Run(models, batch_size, 0.1);\n\n // if testing mps - sync all processes before executing timed loop\n MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));\n auto results = benchmark.Run(models, batch_size, FLAGS_seconds);\n MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));\n // todo: perform an mpi_allreduce to collect the per process timings\n // for a simplified report\n MPI_CHECK(MPI_Finalize());\n\n using namespace trtlab::TensorRT;\n LOG(INFO) << \"Inference Results: \" << (*results)[kBatchesComputed]\n << \" batches computed in \" << (*results)[kWalltime] << \" seconds on \"\n << (*results)[kMaxExecConcurrency]\n << \" compute streams using batch_size: \" << (*results)[kBatchSize]\n << \"; inf/sec: \" << (*results)[kInferencesPerSecond]\n << \"; batches/sec: \" << (*results)[kBatchesPerSecond]\n << \"; execution time per batch: \" << (*results)[kExecutionTimePerBatch];\n }\n\n return 0;\n}\n" }, { "path": "examples/00_TensorRT/inference.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n\n#include \n#include \n\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n#include \"tensorrt/laboratory/inference_manager.h\"\n#include \"tensorrt/laboratory/runtime.h\"\n\n#ifdef PLAYGROUND_USE_MPI\n#include \"mpi.h\"\n#define MPI_CHECK(mpicall) mpicall\n#else\n#define MPI_CHECK(mpicall)\n#endif\n\nusing trtlab::ThreadPool;\nusing trtlab::TensorRT::CustomRuntime;\nusing trtlab::TensorRT::InferenceManager;\nusing trtlab::TensorRT::ManagedAllocator;\nusing trtlab::TensorRT::Runtime;\nusing trtlab::TensorRT::StandardAllocator;\n\nstatic int g_Concurrency = 0;\n\nstatic std::string ModelName(int model_id)\n{\n std::ostringstream stream;\n stream << model_id;\n return stream.str();\n}\n\nclass InferenceResources : public InferenceManager\n{\n public:\n InferenceResources(int max_executions, int max_buffers, size_t nCuda, size_t nResp)\n : InferenceManager(max_executions, max_buffers),\n m_CudaThreadPool(std::make_unique(nCuda)),\n m_ResponseThreadPool(std::make_unique(nResp))\n {\n }\n\n ~InferenceResources() override {}\n\n std::unique_ptr& GetCudaThreadPool() { return m_CudaThreadPool; }\n std::unique_ptr& GetResponseThreadPool() { return m_ResponseThreadPool; }\n\n private:\n std::unique_ptr m_CudaThreadPool;\n std::unique_ptr m_ResponseThreadPool;\n};\n\nclass Inference final\n{\n public:\n Inference(std::shared_ptr resources) : m_Resources(resources) {}\n\n void Run(float seconds, bool warmup, int replicas, uint32_t requested_batch_size)\n {\n int replica = 0;\n uint64_t inf_count = 0;\n\n auto start = std::chrono::steady_clock::now();\n auto elapsed = [start]() -> float {\n return std::chrono::duration(std::chrono::steady_clock::now() - start).count();\n };\n\n auto model = GetResources()->GetModel(ModelName(replica++));\n auto batch_size = requested_batch_size ? requested_batch_size : model->GetMaxBatchSize();\n if(batch_size > model->GetMaxBatchSize())\n {\n LOG(FATAL)\n << \"Requested batch_size greater than allowed by the compiled TensorRT Engine\";\n }\n\n // Inference Loop - Main thread copies, cuda thread launches, response thread completes\n if(!warmup)\n {\n LOG(INFO) << \"-- Inference: Running for ~\" << (int)seconds\n << \" seconds with batch_size \" << batch_size << \" --\";\n }\n\n std::vector> futures;\n\n while(elapsed() < seconds && ++inf_count)\n {\n if(replica >= replicas) replica = 0;\n\n // This thread only async copies buffers H2D\n auto model = GetResources()->GetModel(ModelName(replica++));\n auto buffers = GetResources()->GetBuffers(); // <=== Limited Resource; May Block !!!\n auto bindings = buffers->CreateBindings(model);\n auto promise = std::make_shared>();\n futures.push_back(promise->get_future());\n\n bindings->SetBatchSize(batch_size);\n bindings->CopyToDevice(bindings->InputBindings());\n\n GetResources()->GetCudaThreadPool()->enqueue([this, bindings, promise]() mutable {\n // This thread enqueues two async kernels:\n // 1) TensorRT execution\n // 2) D2H of output tensors\n auto trt = GetResources()->GetExecutionContext(\n bindings->GetModel()); // <=== Limited Resource; May Block !!!\n trt->Infer(bindings);\n bindings->CopyFromDevice(bindings->OutputBindings());\n\n GetResources()->GetResponseThreadPool()->enqueue(\n [bindings, trt, promise]() mutable {\n // This thread waits on the completion of the async compute and the async\n // copy\n trt->Synchronize();\n trt.reset(); // Finished with the Execution Context - Release it to\n // competing threads\n bindings->Synchronize();\n bindings.reset(); // Finished with Buffers - Release it to competing threads\n promise->set_value();\n });\n });\n }\n\n for(const auto& f : futures)\n {\n f.wait();\n }\n\n /*\n // Join worker threads\n if (!warmup)\n GetResources()->GetCudaThreadPool().reset();\n if (!warmup)\n GetResources()->GetResponseThreadPool().reset();\n */\n // End timing and report\n auto total_time = std::chrono::duration(elapsed()).count();\n auto inferences = inf_count * batch_size;\n if(!warmup)\n LOG(INFO) << \"Inference Results: \" << inf_count << \"; batches in \" << total_time\n << \" seconds\"\n << \"; sec/batch/stream: \" << total_time / (inf_count / g_Concurrency)\n << \"; batches/sec: \" << inf_count / total_time\n << \"; inf/sec: \" << inferences / total_time;\n }\n\n protected:\n inline std::shared_ptr GetResources() { return m_Resources; }\n\n private:\n std::shared_ptr m_Resources;\n};\n\nstatic bool ValidateEngine(const char* flagname, const std::string& value)\n{\n struct stat buffer;\n return (stat(value.c_str(), &buffer) == 0);\n}\n\nDEFINE_string(engine, \"/path/to/tensorrt.engine\", \"TensorRT serialized engine\");\nDEFINE_validator(engine, &ValidateEngine);\nDEFINE_string(runtime, \"default\", \"TensorRT Runtime\");\nDEFINE_int32(seconds, 5, \"Approximate number of seconds for the timing loop\");\nDEFINE_int32(contexts, 1, \"Number of Execution Contexts\");\nDEFINE_int32(buffers, 0, \"Number of Buffers (default: 2x contexts)\");\nDEFINE_int32(cudathreads, 1, \"Number Cuda Launcher Threads\");\nDEFINE_int32(respthreads, 1, \"Number Response Sync Threads\");\nDEFINE_int32(replicas, 1, \"Number of Replicas of the Model to load\");\nDEFINE_int32(batch_size, 0, \"Overrides the max batch_size of the provided engine\");\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"TensorRT Inference\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n MPI_CHECK(MPI_Init(&argc, &argv));\n\n auto contexts = g_Concurrency = FLAGS_contexts;\n auto buffers = FLAGS_buffers ? FLAGS_buffers : 2 * FLAGS_contexts;\n\n auto resources = std::make_shared(contexts, buffers, FLAGS_cudathreads,\n FLAGS_respthreads);\n\n std::shared_ptr runtime;\n if(FLAGS_runtime == \"default\")\n {\n runtime = std::make_shared>();\n }\n else if(FLAGS_runtime == \"unified\")\n {\n runtime = std::make_shared>();\n }\n else\n {\n LOG(FATAL) << \"Invalid TensorRT Runtime\";\n }\n\n resources->RegisterModel(\"0\", runtime->DeserializeEngine(FLAGS_engine));\n resources->AllocateResources();\n\n for(int i = 1; i < FLAGS_replicas; i++)\n {\n resources->RegisterModel(ModelName(i), runtime->DeserializeEngine(FLAGS_engine));\n }\n\n Inference inference(resources);\n inference.Run(0.1, true, 1, 0); // warmup\n\n // if testing mps - sync all processes before executing timed loop\n MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));\n inference.Run(FLAGS_seconds, false, FLAGS_replicas, FLAGS_batch_size);\n MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));\n\n // todo: perform an mpi_allreduce to collect the per process timings\n // for a simplified report\n MPI_CHECK(MPI_Finalize());\n return 0;\n}\n" }, { "path": "examples/01_Basic_GRPC/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nadd_executable(echo-grpc.x\n src/server.cpp)\n\ntarget_link_libraries(echo-grpc.x\n nvrpc\n echo-protos\n gflags\n)\n\nadd_executable(echo-client.x\n src/client.cpp)\n\ntarget_link_libraries(echo-client.x\n nvrpc\n echo-protos\n gflags\n)\n\nadd_executable(async-echo-client.x\n src/async_client.cc)\n\ntarget_link_libraries(async-echo-client.x\n nvrpc\n nvrpc-client\n echo-protos\n gflags\n)" }, { "path": "examples/01_Basic_GRPC/README.md", "content": "Simple service to test and stress the core service and request logic.\n\nThe [`server.cc`](examples/01_Basic_GRPC/server.cc) is very well documented and\nshould be used a reference for the gRPC interface provided by the library.\n\n\n" }, { "path": "examples/01_Basic_GRPC/src/async_client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"nvrpc/client/client_unary.h\"\n#include \"nvrpc/client/executor.h\"\n\n#include \"echo.grpc.pb.h\"\n\nusing grpc::Channel;\nusing grpc::ClientContext;\nusing grpc::Status;\nusing simple::Inference;\nusing simple::Input;\nusing simple::Output;\n\nusing nvrpc::client::ClientUnary;\nusing nvrpc::client::Executor;\n\nDEFINE_int32(count, 100, \"number of grpc messages to send\");\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\n\nint main(int argc, char** argv)\n{\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n auto executor = std::make_shared(FLAGS_thread_count);\n\n auto channel = grpc::CreateChannel(\"localhost:50051\", grpc::InsecureChannelCredentials());\n auto stub = Inference::NewStub(channel);\n\n auto infer_prepare_fn = [&stub](::grpc::ClientContext * context, const ::simple::Input& request,\n ::grpc::CompletionQueue* cq) -> auto\n {\n return std::move(stub->PrepareAsyncCompute(context, request, cq));\n };\n\n auto runner = std::make_unique>(infer_prepare_fn, executor);\n\n auto start = std::chrono::steady_clock::now();\n auto elapsed = [start]() -> float {\n return std::chrono::duration(std::chrono::steady_clock::now() - start).count();\n };\n\n for(int i = 0; i < FLAGS_count; i++)\n {\n Input input;\n input.set_batch_id(i);\n runner->Enqueue(std::move(input),\n [i](Input& input, Output& output, ::grpc::Status& status) -> bool {\n CHECK(output.batch_id() == i);\n LOG_FIRST_N(INFO, 20) << \"Check: \" << i;\n return (bool)(output.batch_id() == i);\n });\n }\n std::cout << FLAGS_count << \" queued in \" << elapsed() << \"seconds\" << std::endl;\n executor->ShutdownAndJoin();\n std::cout << FLAGS_count << \" completed in \" << elapsed() << \"seconds\" << std::endl;\n return 0;\n}" }, { "path": "examples/01_Basic_GRPC/src/client.cpp", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"echo.grpc.pb.h\"\n\nusing grpc::Channel;\nusing grpc::ClientContext;\nusing grpc::Status;\nusing simple::Input;\nusing simple::Output;\nusing simple::Inference;\n\nclass SimpleClient {\n public:\n SimpleClient(std::shared_ptr channel)\n : stub_(Inference::NewStub(channel)) {}\n\n // Assembles the client's payload, sends it and presents the response back\n // from the server.\n int Compute(const int batch_id) {\n // Data we are sending to the server.\n Input request;\n request.set_batch_id(batch_id);\n\n // Container for the data we expect from the server.\n Output reply;\n\n // Context for the client. It could be used to convey extra information to\n // the server and/or tweak certain RPC behaviors.\n ClientContext context;\n\n // The actual RPC.\n Status status = stub_->Compute(&context, request, &reply);\n\n // Act upon its status.\n if (status.ok()) {\n return reply.batch_id();\n } else {\n std::cout << status.error_code() << \": \" << status.error_message()\n << std::endl;\n return -1;\n }\n }\n\n private:\n std::unique_ptr stub_;\n};\n\nDEFINE_int32(count, 100, \"number of grpc messages to send\");\n\nint main(int argc, char** argv) {\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n SimpleClient client(grpc::CreateChannel(\n \"localhost:50051\", grpc::InsecureChannelCredentials()));\n auto start = std::chrono::steady_clock::now();\n for(int i=0; i(end - start).count();\n std::cout << FLAGS_count << \" requests in \" << elapsed << \"seconds\" << std::endl;\n return 0;\n}\n" }, { "path": "examples/01_Basic_GRPC/src/server.cpp", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n#include \"nvrpc/executor.h\"\n#include \"tensorrt/laboratory/core/pool.h\"\n#include \"tensorrt/laboratory/core/resources.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n\n#include \"echo.pb.h\"\n#include \"echo.grpc.pb.h\"\n\nusing nvrpc::AsyncService;\nusing nvrpc::AsyncRPC;\nusing nvrpc::Context;\nusing nvrpc::Executor;\nusing nvrpc::Server;\nusing trtlab::Resources;\nusing trtlab::ThreadPool;\n\n// CLI Options\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\n\n/**\n * Embedding a copy of the Protobuf specification for the gRPC service.\n *\n * Package Name: simple\n * Service Name: Inference\n * RPC Name: Compute\n * \n * Incoming Message: Input\n * Outgoing Message: Ouput\n **\n\nsyntax = \"proto3\";\n\npackage simple;\n\nservice Inference {\n rpc Compute (Input) returns (Output) {}\n}\n\nmessage Input {\n uint64 batch_id = 1;\n}\n\nmessage Output {\n uint64 batch_id = 1;\n}\n*/\n\n// Define the resources your RPC will need to execute\n// ==================================================\n// In this case, all simple::Inference::Compute RPCs share a threadpool in which they will\n// queue up some work on. This essentially means, after the message as been received and\n// processed, the actual work for the RPC is pushed to a worker pool outside the scope of\n// the transaction processing system (TPS). This is essentially async computing, we have \n// decoupled the transaction from the workers executing the implementation. The TPS can\n// continue to queue work, while the workers process the load.\nstruct SimpleResources : public Resources\n{\n SimpleResources(int numThreadsInPool=3) : m_ThreadPool(numThreadsInPool) {\n LOG(INFO) << \"Server ThreadCount: \" << numThreadsInPool;\n }\n\n ThreadPool& AcquireThreadPool()\n {\n return m_ThreadPool;\n }\n\n private:\n ThreadPool m_ThreadPool;\n};\n\n// Contexts hold the state and provide the definition of the work to be performed by the RPC.\n// This is where you define what gets executed for a given RPC.\n// Incoming Message = simple::Input (RequestType)\n// Outgoing Message = simple::Output (ResponseType)\nclass SimpleContext final : public Context\n{\n void ExecuteRPC(RequestType &input, ResponseType &output) final override\n {\n // We could do work here, but we'd block the TPS, i.e. the threads pulling messages \n // off the incoming recieve queue. Very quick responses are best done here; however,\n // longer running workload should be offloaded so the TPS can avoid being blocked.\n GetResources()->AcquireThreadPool().enqueue([this, &input, &output]{\n // Now running on a worker thread of the ThreadPool defined in SimpleResources.\n // Here we are just echoing back the incoming // batch_id; however, in later \n // examples, we'll show how to run an async cuda pipline.\n LOG_FIRST_N(INFO, 20) << \"Tag = \" << Tag() << \" Thread = \" << std::this_thread::get_id();\n output.set_batch_id(input.batch_id());\n this->FinishResponse();\n });\n // The TPS thread is now free to continue processing message - async ftw!\n }\n};\n\n\nint main(int argc, char *argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n\n ::google::InitGoogleLogging(\"simpleServer\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n // A server will bind an IP:PORT to listen on\n Server server(\"0.0.0.0:50051\");\n\n // A server can host multiple services\n LOG(INFO) << \"Register Service (simple::Inference) with Server\"; \n auto simpleInference = server.RegisterAsyncService();\n\n // An RPC has two components that need to be specified when registering with the service:\n // 1) Type of Execution Context (SimpleContext). The execution context defines the behavor\n // of the RPC, i.e. it contains the control logic for the execution of the RPC.\n // 2) The Request function (RequestCompute) which was generated by gRPC when compiling the\n // protobuf which defined the service. This function is responsible for queuing the\n // RPC's execution context to the \n LOG(INFO) << \"Register RPC (simple::Inference::Compute) with Service (simple::Inference)\";\n auto rpcCompute = simpleInference->RegisterRPC(\n &simple::Inference::AsyncService::RequestCompute\n );\n\n LOG(INFO) << \"Initializing Resources for RPC (simple::Inference::Compute)\";\n auto rpcResources = std::make_shared(FLAGS_thread_count);\n\n // Create Executors - Executors provide the messaging processing resources for the RPCs\n // Multiple Executors can be registered with a Server. The executor is responsible\n // for pulling incoming message off the receive queue and executing the associated\n // context. By default, an executor only uses a single thread. A typical usecase is\n // an Executor executes a context, which immediate pushes the work to a thread pool.\n // However, for very low-latency messaging, you might want to use a multi-threaded \n // Executor and a Blocking Context - meaning the Context performs the entire RPC function\n // on the Executor's thread.\n LOG(INFO) << \"Creating Executor\";\n auto executor = server.RegisterExecutor(new Executor(1));\n\n // You can register RPC execution contexts from any registered RPC on any executor.\n // The power of that will become clear in later examples. For now, we will register\n // 10 instances of the simple::Inference::Compute RPC's SimpleContext execution context\n // with the Executor.\n LOG(INFO) << \"Creating Execution Contexts for RPC (simple::Inference::Compute) with Executor\";\n executor->RegisterContexts(rpcCompute, rpcResources, 10);\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(2000), []{\n // This is a timeout loop executed every 2seconds\n // Run() with no arguments will run an empty timeout loop every 5 seconds.\n // RunAsync() will return immediately, its your responsibility to ensure the\n // server doesn't go out of scope or a Shutdown will be triggered on your services.\n });\n}\n" }, { "path": "examples/02_TensorRT_GRPC/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nfind_package(prometheus-cpp CONFIG REQUIRED)\nif(prometheus-cpp_FOUND)\n message(STATUS \"Prometheus Metics Enabled\")\nendif(prometheus-cpp_FOUND)\n\n\nadd_executable(inference-grpc.x\n src/metrics.cc\n src/server.cc\n)\ntarget_include_directories(inference-grpc.x\n PUBLIC\n $\n)\ntarget_link_libraries(inference-grpc.x\n trtlab::nvrpc\n trtlab::tensorrt\n prometheus-cpp::prometheus-cpp\n demo-protos\n gflags\n)\n\nadd_executable(client-sync.x\n src/sync-client.cc\n)\ntarget_link_libraries(client-sync.x\n nvrpc\n demo-protos\n gflags\n)\n\nadd_executable(client-async.x\n src/async-client.cc\n)\ntarget_link_libraries(client-async.x\n nvrpc\n demo-protos\n gflags\n)\n\nadd_executable(siege.x\n src/siege.cc\n)\ntarget_link_libraries(siege.x\n nvrpc\n demo-protos\n gflags\n)\n" }, { "path": "examples/02_TensorRT_GRPC/README.md", "content": "# TensorRT GRPC Example\n\nThis examples extends the [TensorRT](examples/00_TensorRT) compute loop into an\nasync gRPC service similar to [example 01_gRPC](examples/01_GRPC).\n\nThere are three take-aways from this example:\n\n1. TensorRT compute pipeline is implemented as the `ExecuteRPC` virtual function\n of the `Context`.\n2. An external datasource is used to override the input bindings\n3. Custom [Prometheus](https://prometheus.io) metrics for inference compute and\n request durations, load ratio, and GPU power gauge.\n are recorded/observed.\n\n## Quickstart\n\n```\ncd /work/build/examples/02_TensorRT_GRPC\n./inference-grpc.x --contexts=8 --engine=/work/models/ResNet-50-b1-int8.engine --port 50051 &\n./siege.x --port=50051 --rate=2500\n# ctrl+c to cancel client\ntelegraf -test -config /work/examples/91_Prometheus/scrape.conf\n```\n\n## Explore\n\nFun things to try:\n\n * Evaluate the performance of the model using `inference.x` in \n [examples/00_TensorRT](examples/00_TensorRT)\n * Try running `siege.x` below, at, and above the benchmarked rate and watch the metrics\n via `telegraf`.\n * Deploy on Kubernetes, collect metrics via Prometheus and visualize using Grafana;\n [examples/90_Kubernetes](examples/90_Kubernetes).\n\n## Server/Service\n\n`inference-grpc.x` CLI options:\n\n * `--engine` - the compiled TensorRT plan/engine\n * `--contexts` - the maximium number of concurrent evaluations of the engine.\n * `--port` - the port on which requests are received (default: 50051)\n * `--metrics` - the port on which to expose metrics to be scraped (default: 50078)\n\n\n## Clients\n\nThree clients are available:\n * `client-sync.x` - sends a blocking inference request to the service and waits for the\n response. Only 1 request is ever in-flight at a given time.\n * `client-async.x` - the async client is capable of issuing multiple in-flight requests.\n Note: the load-balancer is limited to 1000 outstanding requests per client before circuit-\n breaking. Running more than 1000 requests will trigger 503 if targeting the envoy load-\n balancer. The client has no backoff and will try to send the full compliment of requested\n inference requests. `siege.x` is the better async client.\n * `siege.x` - constant rate (`--rate`) async engine that is hard-coded to have no more than\n 950 outstanding in-flight requests. A warning will be given client-side if the outstanding\n requests tops meaning the rate is limited by the server-side compute.\n\nTODO:\n * Add more varied test clients akin to [Netflix's Chaos Monkeys](https://github.com/Netflix/chaosmonkey),\n but for gRPC client behavior.\n * Random rate, random pulses, canceled messsages, messages wiht unreasonable timeouts, etc.\n\n## Metrics\n\nYAIS metrics are gathered and exposed via the [prometheus-cpp](https://github.com/jupp0r/prometheus-cpp) \nclient library. In this example, we expose four custom \n[metrics](https://prometheus.io/docs/concepts/metric_types/): 2 Summaries, 1 Histogram and 1 Gauge.\n\n * `compute_duration` and `request_duration` are summaries recored with the model\n name as a component of the metric. This is useful for evaluating how a given\n model is performing, but this is not a good metric to aggregate across multiple\n service.\n * `load_ratio` is a histogram of `request_duraton / compute_duration`. Ideally, this\n unitless value is just over 1.0. Values higher than 1.0 are indictive of some\n delays in the compute of a given request. Sources of delays include, overloaded\n queues and/or starvation of resources. Histograms can be aggregated across services,\n which makes this metric a good candidate for triggering auto-scaling.\n * `gpu_power` is a simple gauge that periodicly reports the instaneous power being\n consumed by the device. As the load increases on the service, the power should \n increase proprotionally, until the power is capped either by device limits or compute \n resources. When power capped, the `load_ratio` will begin to increase under futher \n increases in traffic.\n\n\n### Acquiring Metrics\n\nPrometheus metrics are generally scraped by a Prometheus service. When using Kubernetes\nto deploy services, the [prometheus-operator](https://github.com/coreos/prometheus-operator)\nprovides a [`ServiceMonitor`](https://github.com/coreos/prometheus-operator#customresourcedefinitions)\nwhich allows you to define custom scraping configuration per service. See the \n[Kubernetes example](examples/90_Kubernetes) for more details.\n\nWhile testing, you can use the [`telegraf`](https://github.com/influxdata/telegraf) application\nto scrape local services.\n\n```\n# start service\ntelegraf -test -config /work/examples/91_Prometheus/scrape.conf\n```\n\nHere is some sample output (line breaks added for readability):\n```\n> yais_inference_compute_duration_ms,host=dgx,model=flowers,url=http://localhost:50078/metrics count=1000,sum=2554.070996 1530985302000000000\n\n> yais_inference_compute_duration_ms_quantile,host=dgx,model=flowers,quantile=0.500000,url=http://localhost:50078/metrics value=2.526903 1530985302000000000\n> yais_inference_compute_duration_ms_quantile,host=dgx,model=flowers,quantile=0.900000,url=http://localhost:50078/metrics value=2.625447 1530985302000000000\n> yais_inference_compute_duration_ms_quantile,host=dgx,model=flowers,quantile=0.990000,url=http://localhost:50078/metrics value=2.855728 1530985302000000000\n\n> yais_inference_request_duration_ms,host=dgx,model=flowers,url=http://localhost:50078/metrics count=1000,sum=243547.558097 1530985302000000000\n> yais_inference_request_duration_ms_quantile,host=dgx,model=flowers,quantile=0.500000,url=http://localhost:50078/metrics value=253.216653 1530985302000000000\n> yais_inference_request_duration_ms_quantile,host=dgx,model=flowers,quantile=0.900000,url=http://localhost:50078/metrics value=256.715759 1530985302000000000\n> yais_inference_request_duration_ms_quantile,host=dgx,model=flowers,quantile=0.990000,url=http://localhost:50078/metrics value=275.407232 1530985302000000000\n\n> yais_inference_load_ratio,host=dgx,url=http://localhost:50078/metrics +Inf=1000,1.25=1,1.5=1,10=9,100=253,2=1,count=1000,sum=95879.013208 1530985302000000000\n\n> yais_gpus_power_usage,gpu=0,host=dgx,url=http://localhost:50078/metrics gauge=52.821 1530985302000000000\n\n> yais_executor_queue_depth,host=dgx,url=http://localhost:50078/metrics gauge=0 1530985302000000000\n```\n### Best Practices\n\nFor a good description of using histograms vs. summaries to collect meaningful metrics\nsee: https://prometheus.io/docs/practices/histograms/\n\nTwo rules of thumb:\n - If you need to aggregate, choose histograms.\n - Otherwise, choose a histogram if you have an idea of the range and distribution of \n values that will be observed. Choose a summary if you need an accurate quantile, no\n matter what the range and distribution of the values is.\n\n" }, { "path": "examples/02_TensorRT_GRPC/src/async-client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n * Original Copyright proivded below.\n * This work extends the original gRPC client examples to work with the\n * implemented server.\n *\n * Copyright 2015 gRPC authors.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n */\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n\n#include \n#include \n#include \n\n#include \"inference.grpc.pb.h\"\n\nusing grpc::Channel;\nusing grpc::ClientAsyncResponseReader;\nusing grpc::ClientContext;\nusing grpc::CompletionQueue;\nusing grpc::Status;\nusing ssd::BatchInput;\nusing ssd::BatchPredictions;\nusing ssd::Inference;\n\nclass GreeterClient\n{\n public:\n explicit GreeterClient(std::shared_ptr channel) : stub_(Inference::NewStub(channel)) {}\n\n // Assembles the client's payload and sends it to the server.\n void SayHello(const size_t batch_id, const int batch_size)\n {\n // Data we are sending to the server.\n BatchInput request;\n request.set_batch_id(batch_id);\n request.set_batch_size(batch_size);\n\n // Call object to store rpc data\n AsyncClientCall* call = new AsyncClientCall;\n\n // stub_->PrepareAsyncSayHello() creates an RPC object, returning\n // an instance to store in \"call\" but does not actually start the RPC\n // Because we are using the asynchronous API, we need to hold on to\n // the \"call\" instance in order to get updates on the ongoing RPC.\n call->response_reader = stub_->PrepareAsyncCompute(&call->context, request, &cq_);\n\n // StartCall initiates the RPC call\n call->response_reader->StartCall();\n\n // Request that, upon completion of the RPC, \"reply\" be updated with the\n // server's response; \"status\" with the indication of whether the operation\n // was successful. Tag the request with the memory address of the call object.\n call->response_reader->Finish(&call->reply, &call->status, (void*)call);\n }\n\n // Loop while listening for completed responses.\n // Prints out the response from the server.\n void AsyncCompleteRpc()\n {\n void* got_tag;\n bool ok = false;\n\n // Block until the next result is available in the completion queue \"cq\".\n while(cq_.Next(&got_tag, &ok))\n {\n // The tag in this example is the memory location of the call object\n AsyncClientCall* call = static_cast(got_tag);\n\n // Verify that the request was completed successfully. Note that \"ok\"\n // corresponds solely to the request for updates introduced by Finish().\n GPR_ASSERT(ok);\n\n if(call->status.ok())\n {\n // std::cout << \"Greeter received: \" << call->reply.batch_id() << std::endl;\n }\n else\n {\n std::cout << \"RPC failed\" << std::endl;\n }\n // Once we're complete, deallocate the call object.\n delete call;\n }\n }\n\n void Shutdown() { cq_.Shutdown(); }\n\n private:\n // struct for keeping state and data information\n struct AsyncClientCall\n {\n // Container for the data we expect from the server.\n BatchPredictions reply;\n\n // Context for the client. It could be used to convey extra information to\n // the server and/or tweak certain RPC behaviors.\n ClientContext context;\n\n // Storage for the status of the RPC upon completion.\n Status status;\n\n std::unique_ptr> response_reader;\n };\n\n // Out of the passed in Channel comes the stub, stored here, our view of the\n // server's exposed services.\n std::unique_ptr stub_;\n\n // The producer-consumer queue we use to communicate asynchronously with the\n // gRPC runtime.\n CompletionQueue cq_;\n};\n\nDEFINE_int32(count, 500, \"number of grpc messages to send\");\nDEFINE_int32(batch_size, 1, \"batch_size\");\nDEFINE_int32(port, 50051, \"server_port\");\n\nint main(int argc, char** argv)\n{\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n std::ostringstream ip_port;\n ip_port << \"localhost:\" << FLAGS_port;\n GreeterClient greeter(grpc::CreateChannel(ip_port.str(), grpc::InsecureChannelCredentials()));\n\n // Spawn reader thread that loops indefinitely\n std::thread thread_ = std::thread(&GreeterClient::AsyncCompleteRpc, &greeter);\n\n auto start = std::chrono::steady_clock::now();\n for(size_t i = 0; i < FLAGS_count; i++)\n {\n greeter.SayHello(i, FLAGS_batch_size); // The actual RPC call!\n }\n\n greeter.Shutdown();\n thread_.join(); // blocks forever\n auto end = std::chrono::steady_clock::now();\n float elapsed = std::chrono::duration(end - start).count();\n std::cout << FLAGS_count << \" requests in \" << elapsed\n << \"seconds; inf/sec: \" << FLAGS_count * FLAGS_batch_size / elapsed << std::endl;\n\n return 0;\n}\n" }, { "path": "examples/02_TensorRT_GRPC/src/metrics.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \"metrics.h\"\n\n#include \n#include \n\nnamespace trtlab {\n\nvoid Metrics::Initialize(uint32_t port)\n{\n auto singleton = GetSingleton();\n if(singleton->m_Exposer)\n {\n LOG(WARNING) << \"Metrics already initialized. This call is ignored\";\n return;\n }\n std::ostringstream stream;\n stream << \"0.0.0.0:\" << port;\n singleton->m_Exposer = std::make_unique(stream.str());\n singleton->m_Exposer->RegisterCollectable(singleton->m_Registry);\n}\n\nauto Metrics::GetRegistry() -> Registry&\n{\n auto singleton = Metrics::GetSingleton();\n return *(singleton->m_Registry);\n}\n\nMetrics* Metrics::GetSingleton()\n{\n static Metrics singleton;\n return &singleton;\n}\n\nMetrics::Metrics() : m_Registry(std::make_shared()) {}\n\nMetrics::~Metrics() {}\n\n} // namespace trtlab" }, { "path": "examples/02_TensorRT_GRPC/src/metrics.h", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#pragma once\n\n#include \n#include \n\nusing prometheus::Exposer;\nusing prometheus::Registry;\n\nnamespace trtlab {\n\nclass Metrics\n{\n public:\n static void Initialize(uint32_t port);\n static auto GetRegistry() -> Registry&;\n\n protected:\n Metrics();\n virtual ~Metrics();\n static Metrics* GetSingleton();\n\n private:\n std::unique_ptr m_Exposer;\n std::shared_ptr m_Registry;\n};\n\n} // namespace trtlab\n" }, { "path": "examples/02_TensorRT_GRPC/src/server.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \"nvml.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"tensorrt/laboratory/core/affinity.h\"\n#include \"tensorrt/laboratory/core/memory/allocator.h\"\n#include \"tensorrt/laboratory/cuda/device_info.h\"\n#include \"tensorrt/laboratory/cuda/memory/cuda_pinned_host.h\"\n#include \"tensorrt/laboratory/inference_manager.h\"\n#include \"tensorrt/laboratory/runtime.h\"\n\n#include \"nvrpc/context.h\"\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n\n#include \"metrics.h\"\n\nusing nvrpc::AsyncRPC;\nusing nvrpc::AsyncService;\nusing nvrpc::Context;\nusing nvrpc::Executor;\nusing nvrpc::Server;\nusing trtlab::Affinity;\nusing trtlab::Allocator;\nusing trtlab::CudaPinnedHostMemory;\nusing trtlab::DeviceInfo;\nusing trtlab::Metrics;\nusing trtlab::ThreadPool;\nusing trtlab::TensorRT::InferenceManager;\nusing trtlab::TensorRT::ManagedRuntime;\nusing trtlab::TensorRT::Model;\nusing trtlab::TensorRT::Runtime;\nusing trtlab::TensorRT::StandardRuntime;\n\n// Flowers Protos\n#include \"inference.grpc.pb.h\"\n#include \"inference.pb.h\"\n\nusing ssd::BatchInput;\nusing ssd::BatchPredictions;\nusing ssd::Inference;\n\n/*\n * Prometheus Metrics\n *\n * It is important to make collect measurements to find bottlenecks, performance issues,\n * and to trigger auto-scaling.\n */\nstatic auto& registry = Metrics::GetRegistry();\n\n// Summaries - Request and Compute duration on a per service basis\nstatic auto& inf_compute =\n prometheus::BuildSummary().Name(\"yais_inference_compute_duration_ms\").Register(registry);\nstatic auto& inf_request =\n prometheus::BuildSummary().Name(\"yais_inference_request_duration_ms\").Register(registry);\nstatic const auto& quantiles =\n prometheus::Summary::Quantiles{{0.5, 0.05}, {0.90, 0.01}, {0.99, 0.001}};\n\n// Histogram - Load Ratio = Request/Compute duration - should just above one for a service\n// that can keep up with its current load. This metrics provides more\n// detailed information on the impact of the queue depth because it accounts\n// for request time.\nstatic const std::vector buckets = {1.25, 1.50, 2.0, 10.0, 100.0}; // unitless\nstatic auto& inf_load_ratio_fam =\n prometheus::BuildHistogram().Name(\"yais_inference_load_ratio\").Register(registry);\nstatic auto& inf_load_ratio = inf_load_ratio_fam.Add({}, buckets);\n\n// Gauge - Periodically measure and report GPU power utilization. As the load increases\n// on the service, the power should increase proprotionally, until the power is capped\n// either by device limits or compute resources. At this level, the inf_load_ratio\n// will begin to increase under futher increases in traffic\nstatic auto& power_gauge_fam =\n prometheus::BuildGauge().Name(\"yais_gpus_power_usage\").Register(registry);\nstatic auto& power_gauge = power_gauge_fam.Add({{\"gpu\", \"0\"}});\n\n/*\n * External Data Source\n *\n * Attaches to a System V shared memory segment owned by an external resources.\n * Example: the results of an image decode service could use this mechanism to transfer\n * large tensors to an inference service by simply passing an offset.\n */\nfloat* GetSharedMemory(const std::string& address);\n\n/*\n * YAIS Resources - TensorRT InferenceManager + ThreadPools + External Datasource\n */\nclass FlowersResources : public InferenceManager\n{\n public:\n explicit FlowersResources(int max_executions, int max_buffers, int nCuda, int nResp,\n float* sysv_data)\n : InferenceManager(max_executions, max_buffers), m_CudaThreadPool(nCuda),\n m_ResponseThreadPool(nResp), m_SharedMemory(sysv_data)\n {\n }\n\n ThreadPool& GetCudaThreadPool() { return m_CudaThreadPool; }\n ThreadPool& GetResponseThreadPool() { return m_ResponseThreadPool; }\n\n float* GetSysvOffset(size_t offset_in_bytes)\n {\n return &m_SharedMemory[offset_in_bytes / sizeof(float)];\n }\n\n private:\n ThreadPool m_CudaThreadPool;\n ThreadPool m_ResponseThreadPool;\n float* m_SharedMemory;\n};\n\n/*\n * nvRPC Context - Defines the logic of the RPC.\n */\nclass FlowersContext final : public Context\n{\n void ExecuteRPC(RequestType& input, ResponseType& output) final override\n {\n // Executing on a Executor threads - we don't want to block message handling, so we offload\n GetResources()->GetCudaThreadPool().enqueue([this, &input, &output]() {\n // Executed on a thread from CudaThreadPool\n auto model = GetResources()->GetModel(\"flowers\");\n auto buffers = GetResources()->GetBuffers(); // <=== Limited Resource; May Block !!!\n auto bindings = buffers->CreateBindings(model);\n bindings->SetBatchSize(input.batch_size());\n bindings->SetHostAddress(0, GetResources()->GetSysvOffset(input.sysv_offset()));\n bindings->CopyToDevice(bindings->InputBindings());\n auto ctx =\n GetResources()->GetExecutionContext(model); // <=== Limited Resource; May Block !!!\n ctx->Infer(bindings);\n bindings->CopyFromDevice(bindings->OutputBindings());\n // All Async CUDA work has been queued - this thread's work is done.\n GetResources()->GetResponseThreadPool().enqueue([this, &input, &output, model, bindings,\n ctx]() mutable {\n // Executed on a thread from ResponseThreadPool\n auto compute_time = ctx->Synchronize();\n ctx.reset(); // Finished with the Execution Context - Release it to competing\n // threads\n bindings->Synchronize(); // Blocks on H2D, Compute, D2H Pipeline\n WriteBatchPredictions(input, output, (float*)bindings->HostAddress(1));\n bindings.reset(); // Finished with Buffers - Release it to competing threads\n auto request_time = Walltime();\n output.set_compute_time(static_cast(compute_time));\n output.set_total_time(static_cast(request_time));\n this->FinishResponse();\n // The Response is now sending; Record some metrics and be done\n inf_compute.Add({{\"model\", model->Name()}}, quantiles).Observe(compute_time * 1000);\n inf_request.Add({{\"model\", model->Name()}}, quantiles).Observe(request_time * 1000);\n inf_load_ratio.Observe(request_time / compute_time);\n });\n });\n }\n\n void WriteBatchPredictions(RequestType& input, ResponseType& output, float* scores)\n {\n int N = input.batch_size();\n auto nClasses = GetResources()->GetModel(\"flowers\")->GetBinding(1).elementsPerBatchItem;\n size_t cntr = 0;\n for(int p = 0; p < N; p++)\n {\n auto element = output.add_elements();\n /* Customize the post-processing of the output tensor *\\\n float max_val = -1.0;\n int max_idx = -1;\n for (int i = 0; i < nClasses; i++)\n {\n if (max_val < scores[cntr])\n {\n max_val = scores[cntr];\n max_idx = i;\n }\n cntr++;\n }\n auto top1 = element->add_predictions();\n top1->set_class_id(max_idx);\n top1->set_score(max_val);\n \\* Customize the post-processing of the output tensor */\n }\n output.set_batch_id(input.batch_id());\n }\n};\n\nstatic bool ValidateEngine(const char* flagname, const std::string& value)\n{\n struct stat buffer;\n return (stat(value.c_str(), &buffer) == 0);\n}\n\nstatic bool ValidateBytes(const char* flagname, const std::string& value)\n{\n trtlab::StringToBytes(value);\n return true;\n}\n\nDEFINE_string(engine, \"/path/to/tensorrt.engine\", \"TensorRT serialized engine\");\nDEFINE_validator(engine, &ValidateEngine);\nDEFINE_string(dataset, \"127.0.0.1:4444\", \"GRPC Dataset/SharedMemory Service Address\");\nDEFINE_int32(contexts, 1, \"Number of Execution Contexts\");\nDEFINE_int32(buffers, 0, \"Number of Input/Output Buffers\");\nDEFINE_string(runtime, \"default\", \"TensorRT Runtime\");\nDEFINE_int32(execution_threads, 1, \"Number of RPC execution threads\");\nDEFINE_int32(preprocessing_threads, 0, \"Number of preprocessing threads\");\nDEFINE_int32(kernel_launching_threads, 1, \"Number of threads to launch CUDA kernels\");\nDEFINE_int32(postprocessing_threads, 2, \"Number of postprocessing threads\");\nDEFINE_string(max_recv_bytes, \"10MiB\", \"Maximum number of bytes for incoming messages\");\nDEFINE_validator(max_recv_bytes, &ValidateBytes);\nDEFINE_int32(port, 50051, \"Port to listen for gRPC requests\");\nDEFINE_int32(metrics, 50078, \"Port to expose metrics for scraping\");\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"flowers\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n // Set CPU Affinity to be near the GPU\n auto cpus = DeviceInfo::Affinity(0);\n Affinity::SetAffinity(cpus);\n\n // Enable metrics on port\n Metrics::Initialize(FLAGS_metrics);\n\n // Create a gRPC server bound to IP:PORT\n std::ostringstream ip_port;\n ip_port << \"0.0.0.0:\" << FLAGS_port;\n Server server(ip_port.str());\n\n // Modify MaxReceiveMessageSize\n auto bytes = trtlab::StringToBytes(FLAGS_max_recv_bytes);\n server.Builder().SetMaxReceiveMessageSize(bytes);\n LOG(INFO) << \"gRPC MaxReceiveMessageSize = \" << trtlab::BytesToString(bytes);\n\n // A server can host multiple services\n LOG(INFO) << \"Register Service (flowers::Inference) with Server\";\n auto inferenceService = server.RegisterAsyncService();\n\n // An RPC has two components that need to be specified when registering with the service:\n // 1) Type of Execution Context (FlowersContext). The execution context defines the behavor\n // of the RPC, i.e. it contains the control logic for the execution of the RPC.\n // 2) The Request function (RequestCompute) which was generated by gRPC when compiling the\n // protobuf which defined the service. This function is responsible for queuing the\n // RPC's execution context to the\n LOG(INFO) << \"Register RPC (flowers::Inference::Compute) with Service (flowers::Inference)\";\n auto rpcCompute =\n inferenceService->RegisterRPC(&Inference::AsyncService::RequestCompute);\n\n // Buffers default to execution contexts + 2\n // Allows for 1 H2D, N TensorRT Executions, 1 D2H to be inflight\n auto buffers = FLAGS_buffers;\n if(buffers == 0) buffers = FLAGS_contexts + 2;\n\n // Initialize Resources\n LOG(INFO) << \"Initializing Resources for RPC (flowers::Inference::Compute)\";\n auto rpcResources = std::make_shared(\n FLAGS_contexts, // number of IExecutionContexts - scratch space for DNN activations\n buffers, // number of host/device buffers for input/output tensors\n FLAGS_kernel_launching_threads, // number of threads used to execute cuda kernel launches\n FLAGS_postprocessing_threads, // number of threads used to write and complete responses\n GetSharedMemory(FLAGS_dataset) // pointer to data in shared memory\n );\n\n std::shared_ptr runtime;\n if(FLAGS_runtime == \"default\")\n {\n runtime = std::make_shared();\n }\n else if(FLAGS_runtime == \"unified\")\n {\n runtime = std::make_shared();\n }\n else\n {\n LOG(FATAL) << \"Invalid TensorRT Runtime\";\n }\n\n rpcResources->RegisterModel(\"flowers\", runtime->DeserializeEngine(FLAGS_engine));\n rpcResources->AllocateResources();\n\n // Create Executors - Executors provide the messaging processing resources for the RPCs\n LOG(INFO) << \"Initializing Executor\";\n auto executor = server.RegisterExecutor(new Executor(1));\n\n // You can register RPC execution contexts from any registered RPC on any executor.\n LOG(INFO)\n << \"Registering Execution Contexts for RPC (flowers::Inference::Compute) with Executor\";\n executor->RegisterContexts(rpcCompute, rpcResources, 100);\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(2000), [] {\n // Query GPU Power\n nvmlDevice_t gpu;\n unsigned int power;\n CHECK_EQ(nvmlDeviceGetHandleByIndex(0, &gpu), NVML_SUCCESS)\n << \"Failed to get Device for index=\" << 0;\n CHECK_EQ(nvmlDeviceGetPowerUsage(gpu, &power), NVML_SUCCESS)\n << \"Failed to get Power Usage for GPU=\" << 0;\n power_gauge.Set((double)power * 0.001);\n });\n}\n\nstatic auto pinned_memory = std::make_unique>(1024 * 1024 * 1024);\n\nfloat* GetSharedMemory(const std::string& address)\n{\n /* data in shared memory should go here - for the sake of quick examples just use and emptry\n * array */\n pinned_memory->Fill((char)0);\n return (float*)pinned_memory->Data();\n // the following code connects to a shared memory service to allow for non-serialized transfers\n // between microservices\n /*\n InfoRequest request;\n Info reply;\n grpc::ClientContext context;\n auto channel = grpc::CreateChannel(address, grpc::InsecureChannelCredentials());\n auto stub = SharedMemoryDataSet::NewStub(channel);\n auto status = stub->GetInfo(&context, request, &reply);\n CHECK(status.ok()) << \"Dataset shared memory request failed\";\n DLOG(INFO) << \"SysV ShmKey: \" << reply.sysv_key();\n int shmid = shmget(reply.sysv_key(), 0, 0);\n DLOG(INFO) << \"SysV ShmID: \" << shmid;\n float* data = (float*) shmat(shmid, 0, 0);\n CHECK(data) << \"SysV Attached failed\";\n return data;\n */\n}\n" }, { "path": "examples/02_TensorRT_GRPC/src/siege.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n * Original Copyright proivded below.\n * This work extends the original gRPC client examples to work with the\n * implemented server.\n *\n * Copyright 2015 gRPC authors.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n */\n\n#include \n#include \n#include \n#include \n\n#include \n#include \n\n#include \n#include \n#include \n\n#include \"inference.grpc.pb.h\"\n\n#include \"tensorrt/laboratory/core/utils.h\"\n\nusing grpc::Channel;\nusing grpc::ClientAsyncResponseReader;\nusing grpc::ClientContext;\nusing grpc::CompletionQueue;\nusing grpc::Status;\nusing ssd::BatchInput;\nusing ssd::BatchPredictions;\nusing ssd::Inference;\n\nstatic int g_BatchSize = 1;\n\nclass GreeterClient\n{\n public:\n explicit GreeterClient(std::shared_ptr channel, int max_outstanding)\n : stub_(Inference::NewStub(channel)), m_OutstandingMessageCount(0),\n m_MaxOutstandingMessageCount(max_outstanding)\n {\n }\n\n // Assembles the client's payload and sends it to the server.\n void SayHello(const size_t batch_id, const int batch_size, char* bytes, uint64_t total)\n {\n // Data we are sending to the server.\n {\n std::unique_lock lock(m_Mutex);\n m_OutstandingMessageCount++;\n while(m_OutstandingMessageCount >= m_MaxOutstandingMessageCount)\n {\n LOG_FIRST_N(WARNING, 10) << \"Initiated Backoff - (Siege Rate > Server Compute \"\n \"Rate) - Server Queues are full.\";\n m_Condition.wait(lock);\n }\n }\n\n auto start = std::chrono::high_resolution_clock::now();\n\n BatchInput request;\n request.set_batch_id(batch_id);\n request.set_batch_size(batch_size);\n if(total)\n {\n request.set_data(bytes, total);\n }\n\n // Call object to store rpc data\n AsyncClientCall* call = new AsyncClientCall;\n\n // stub_->PrepareAsyncSayHello() creates an RPC object, returning\n // an instance to store in \"call\" but does not actually start the RPC\n // Because we are using the asynchronous API, we need to hold on to\n // the \"call\" instance in order to get updates on the ongoing RPC.\n call->response_reader = stub_->PrepareAsyncCompute(&call->context, request, &cq_);\n\n // StartCall initiates the RPC call\n call->response_reader->StartCall();\n\n // Request that, upon completion of the RPC, \"reply\" be updated with the\n // server's response; \"status\" with the indication of whether the operation\n // was successful. Tag the request with the memory address of the call object.\n call->response_reader->Finish(&call->reply, &call->status, (void*)call);\n\n float elapsed =\n std::chrono::duration(std::chrono::high_resolution_clock::now() - start).count();\n m_RequestCalls++;\n m_TotalRequestTime += elapsed;\n // LOG_EVERY_N(INFO, 200) << \"Request overhead: \" << m_TotalRequestTime/m_RequestCalls;\n }\n\n // Loop while listening for completed responses.\n // Prints out the response from the server.\n void AsyncCompleteRpc()\n {\n void* got_tag;\n bool ok = false;\n size_t cntr = 0;\n auto start = std::chrono::steady_clock::now();\n float last = 0.0;\n\n // Block until the next result is available in the completion queue \"cq\".\n while(cq_.Next(&got_tag, &ok))\n {\n // The tag in this example is the memory location of the call object\n AsyncClientCall* call = static_cast(got_tag);\n\n // Verify that the request was completed successfully. Note that \"ok\"\n // corresponds solely to the request for updates introduced by Finish().\n GPR_ASSERT(ok);\n\n if(call->status.ok())\n {\n // std::cout << \"Greeter received: \" << call->reply.batch_id() << std::endl;\n }\n else\n {\n std::cout << \"RPC failed\" << std::endl;\n }\n // Once we're complete, deallocate the call object.\n delete call;\n\n cntr++;\n float elapsed =\n std::chrono::duration(std::chrono::steady_clock::now() - start).count();\n if(elapsed - last > 0.5)\n {\n LOG(INFO) << \"avg. rate: \" << (float)cntr / (elapsed - last) << \"( \"\n << (float)(cntr * g_BatchSize) / (elapsed - last) << \" inf/sec)\";\n last = elapsed;\n cntr = 0;\n }\n\n {\n std::unique_lock lock(m_Mutex);\n m_OutstandingMessageCount--;\n }\n m_Condition.notify_one();\n }\n }\n\n void Shutdown() { cq_.Shutdown(); }\n\n private:\n // struct for keeping state and data information\n struct AsyncClientCall\n {\n // Container for the data we expect from the server.\n BatchPredictions reply;\n\n // Context for the client. It could be used to convey extra information to\n // the server and/or tweak certain RPC behaviors.\n ClientContext context;\n\n // Storage for the status of the RPC upon completion.\n Status status;\n\n std::unique_ptr> response_reader;\n };\n\n // Out of the passed in Channel comes the stub, stored here, our view of the\n // server's exposed services.\n std::unique_ptr stub_;\n\n // The producer-consumer queue we use to communicate asynchronously with the\n // gRPC runtime.\n CompletionQueue cq_;\n\n // mutex to help control rate\n std::mutex m_Mutex;\n std::condition_variable m_Condition;\n int m_OutstandingMessageCount;\n int m_MaxOutstandingMessageCount;\n float m_TotalRequestTime;\n size_t m_RequestCalls;\n};\n\nstatic bool ValidateBytes(const char* flagname, const std::string& value)\n{\n trtlab::StringToBytes(value);\n return true;\n}\n\nDEFINE_int32(count, 1000000, \"number of grpc messages to send\");\nDEFINE_int32(batch_size, 1, \"batch_size\");\nDEFINE_int32(max_outstanding, 950, \"maximum outstanding requests\");\nDEFINE_int32(port, 50051, \"server_port\");\nDEFINE_double(rate, 1.0, \"messages per second\");\nDEFINE_double(max_rate, 100000, \"maximum number of messages per second when func is applied\");\nDEFINE_double(alpha, 0, \"alpha\");\nDEFINE_double(beta, 1, \"beta\");\nDEFINE_string(func, \"constant\", \"constant, linear or cyclic\");\nDEFINE_string(bytes, \"0B\", \"add extra bytes to the request payload\");\nDEFINE_validator(bytes, &ValidateBytes);\n\nint main(int argc, char** argv)\n{\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n g_BatchSize = FLAGS_batch_size;\n\n auto bytes = trtlab::StringToBytes(FLAGS_bytes);\n char extra_bytes[bytes];\n if(bytes)\n LOG(INFO) << \"Sending an addition \" << trtlab::BytesToString(bytes)\n << \" bytes in request payload\";\n\n // using a fixed rate of 15us per rpc call. i could adjust dynamically as i'm tracking\n // the call overhead, but it's close enough.\n auto start = std::chrono::system_clock::now();\n auto walltime = [start]() -> double {\n return std::chrono::duration(std::chrono::system_clock::now() - start).count();\n };\n std::map> rates_by_name;\n rates_by_name[\"constant\"] = []() -> double { return std::min(FLAGS_rate, FLAGS_max_rate); };\n rates_by_name[\"linear\"] = [start, walltime]() -> double {\n return std::min(FLAGS_rate + (FLAGS_alpha / 60.0) * walltime(), FLAGS_max_rate);\n };\n rates_by_name[\"cyclic\"] = [start, walltime]() -> double {\n return std::min(FLAGS_rate + FLAGS_alpha *\n std::sin(2.0 * 3.14159 * (FLAGS_beta / 60.0) * walltime()),\n FLAGS_max_rate);\n };\n auto search = rates_by_name.find(FLAGS_func);\n if(search == rates_by_name.end())\n {\n LOG(FATAL) << \"--func must be constant, linear or cyclic; your value = \" << FLAGS_func;\n }\n auto sleepy = [search]() -> double {\n auto sleep_time = ((std::chrono::seconds(1) / std::max((search->second)(), 2.0))) -\n std::chrono::microseconds(15);\n return std::chrono::duration(sleep_time).count();\n };\n\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n std::ostringstream ip_port;\n ip_port << \"localhost:\" << FLAGS_port;\n\n grpc::ChannelArguments ch_args;\n ch_args.SetMaxReceiveMessageSize(-1);\n GreeterClient greeter(\n grpc::CreateCustomChannel(ip_port.str(), grpc::InsecureChannelCredentials(), ch_args),\n FLAGS_max_outstanding);\n\n // Spawn reader thread that loops indefinitely\n std::thread thread_ = std::thread(&GreeterClient::AsyncCompleteRpc, &greeter);\n\n for(size_t i = 0; i < FLAGS_count; i++)\n {\n greeter.SayHello(i, FLAGS_batch_size, extra_bytes, bytes); // The actual RPC call!\n auto start = std::chrono::high_resolution_clock::now();\n while(std::chrono::duration(std::chrono::high_resolution_clock::now() - start)\n .count() < sleepy())\n {\n std::this_thread::yield();\n }\n }\n\n greeter.Shutdown();\n thread_.join(); // blocks forever\n auto elapsed = walltime();\n std::cout << FLAGS_count << \" requests in \" << elapsed\n << \"seconds; inf/sec: \" << FLAGS_count * FLAGS_batch_size / elapsed << std::endl;\n\n return 0;\n}\n" }, { "path": "examples/02_TensorRT_GRPC/src/sync-client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n * Original Copyright proivded below.\n * This work extends the original gRPC client examples to work with the\n * implemented server.\n *\n * Copyright 2015 gRPC authors.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n */\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"inference.grpc.pb.h\"\n\nusing grpc::Channel;\nusing grpc::ClientContext;\nusing grpc::Status;\nusing ssd::BatchInput;\nusing ssd::BatchPredictions;\nusing ssd::Inference;\n\nclass SimpleClient\n{\n public:\n SimpleClient(std::shared_ptr channel) : stub_(Inference::NewStub(channel)) {}\n\n // Assembles the client's payload, sends it and presents the response back\n // from the server.\n int Compute(const int batch_id, const int batch_size)\n {\n // Data we are sending to the server.\n BatchInput request;\n request.set_batch_id(batch_id);\n request.set_batch_size(batch_size);\n\n // Container for the data we expect from the server.\n BatchPredictions reply;\n\n // Context for the client. It could be used to convey extra information to\n // the server and/or tweak certain RPC behaviors.\n ClientContext context;\n\n // The actual RPC.\n Status status = stub_->Compute(&context, request, &reply);\n\n // Act upon its status.\n if(status.ok())\n {\n return reply.batch_id();\n }\n else\n {\n std::cout << status.error_code() << \": \" << status.error_message() << std::endl;\n return -1;\n }\n }\n\n private:\n std::unique_ptr stub_;\n};\n\nDEFINE_int32(count, 1000, \"number of grpc messages to send\");\nDEFINE_int32(port, 50051, \"server_port\");\nDEFINE_int32(batch, 1, \"batch size\");\n\nint main(int argc, char** argv)\n{\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n std::ostringstream ip_port;\n ip_port << \"localhost:\" << FLAGS_port;\n SimpleClient client(grpc::CreateChannel(ip_port.str(), grpc::InsecureChannelCredentials()));\n auto start = std::chrono::steady_clock::now();\n for(int i = 0; i < FLAGS_count; i++)\n {\n auto reply = client.Compute(i, FLAGS_batch);\n if(reply == -1 || reply != i) std::cout << \"BatchId received: \" << reply << std::endl;\n }\n auto end = std::chrono::steady_clock::now();\n float elapsed = std::chrono::duration(end - start).count();\n std::cout << FLAGS_count << \" requests in \" << elapsed\n << \" seconds; inf/sec: \" << FLAGS_count * FLAGS_batch / elapsed << std::endl;\n return 0;\n}\n" }, { "path": "examples/03_Batching/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nadd_executable(streaming-service-echo.x\n streaming-service.cc\n)\ntarget_link_libraries(streaming-service-echo.x\n trtlab::nvrpc\n echo-protos\n gflags\n)\n\nadd_executable(batching-service-echo.x\n inference-batcher.cc\n)\n\ntarget_link_libraries(batching-service-echo.x\n trtlab::nvrpc\n echo-protos\n gflags\n)\n\n" }, { "path": "examples/03_Batching/README.md", "content": "# Batching Service\n\nA batching service is a service that trying to collect sets of similar requests into a\ncollective batch which can be executed in a single-shot.\n\n#### Why do we want to batch?\n\nIn the case of Deep Neural Networks, batching can improve the computational efficiency\nof executing on a GPU by increases the operational intensity, i.e. improving the ratio of \nthe number of math operations per memory transaction. This translates to improved\nthroughput, better hardware utilization and cost reductions.\n\n#### Sounds great, but what's the catch?\n\nIn many cases, batching can add latency to an individual request. Because a batch of more\nthan 1 item, BatchN, computed as a single unit, the time to compute BatchN is greater than\nBatch1. However, in many cases, the time to compute delta between Batch1 and Batch2/4/8 is \nfairly small due to the improved operational efficiency.\n\nSecondly, because batching requires requests to be collected, there is a timed collection\nwindow prior to the compute. The first request in a batch sees the longest latency. \n\nThe worst-case increased latency is bounded by the following formula:\n```\nworst_additional_latency = batch_window_timeout + batchN_compute - batch1_compute\n```\n\n#### When to Batch?\n\nYou want to batch requests when your service has very high-load and you can tolerate\nminor increases in latency.\n\nThroughput improvements can be 2-5x which translates into direct cost savings.\n\n#### What does this Batching Service do for me?\n\nThe basis YAIS service examples [01_GRPC](../01_GRPC) and [02_GRPC_TensorRT](../02_GRPC_TensorRT)\nhave implemented high-performance send/recv unary services. That is, the client\nsends a request which is computed and a response is returned. The client could in theory\ncreate a single message is itself a batch, i.e. multiple images files or sentences to be \ntranslated. However, in most common realworld usecases, the clients of a service send\na single item at a time. This keeps the logic simple and lifecycle of the request simple.\n\nIf this is your RPC definition,\n```\nservice Inference {\n rpc Compute (Input) returns (Output) {}\n}\n```\n\nThen, instead of implementing `rpc Compute` to perform the inference computation, instead, we\nhyjack that RPC and turn it into a batcher. In the [`inference-batcher.cc`](inference-batcher.cc)\nfile, you will see that we are indeed we implement our batching service as the `Compute` method.\n\nThe batching service collecting incoming `Input` requests and forwards them via a gRPC stream\nto a service that accepts a \"batching stream\".\n\nA “batching stream” is a stream where the endpoint service reads and collects the elements of the stream until the client signifies it is done writing. That is the signal at which YAIS performs a single batched inference call on the concatenated set of requests that came in over the stream. After the inference calculation is complete, the server writes the results for each request item to the stream. That is, for each request that came in on the stream, the server is expected to return a response. \n\nWe still need to compute inference on the batching stream. This is performed by [streaming-service.cc](streaming-service.cc).\n\nThe `streaming-service` implements the `BatchedCompute` RPC method using a `BatchingContext`.\n```\nservice Inference {\n rpc Compute (Input) returns (Output) {}\n rpc BatchedCompute (stream Input) returns (stream Output) {}\n}\n```\n\nBecause the stream consists of an array of individual messages, you simply need to make\nminor modifications to your existing Batch1 service to preprocess and concat the incoming requests\ntogether to form a single batch compute. For each `Input` item in the stream, it is expected that\nthe service writes an `Output` response in the same order as the inputs (FIFO).\n\nThe batching service doesn’t need to know anything about the format of the `Input`/`Output` messages. It simply accepts and forwards them. The result is that this batching service example should be able to work with any unary gRPC service with any request/response message. You simply need to implement a streaming service capable of handling the forwarding stream.\n\n## Running Example\n\n```\n./launch_batching.sh\n```\n\n```\n... # streaming service startup\n... # batching service startup\n\nStarting a shell keeping the services and load-balancer running...\nTry python unary_client.py - exit shell to kill services\nBatching Subshell: python unary_client.py\nI0822 14:48:18.900671 50 inference-batcher.cc:344] incoming unary request\nI0822 14:48:18.902642 41 inference-batcher.cc:109] Client using CQ: 0x14470f0\nI0822 14:48:18.902680 41 inference-batcher.cc:140] Starting Batch Forwarding of Size 1 for Tag 0x1458450\nI0822 14:48:18.903472 35 streaming-service.cc:61] Recieved request with batch_id=78\nI0822 14:48:18.903504 35 streaming-service.cc:54] Response with batch_id=78\nI0822 14:48:18.903656 47 inference-batcher.cc:243] Batch Forwarding Completed for Tag 0x1458450\nReceived msg with batch_id=78\n```\n" }, { "path": "examples/03_Batching/inference-batcher.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \"nvrpc/context.h\"\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n\nusing nvrpc::Context;\nusing nvrpc::Executor;\nusing nvrpc::Server;\nusing trtlab::ThreadPool;\n\n#include \"moodycamel/blockingconcurrentqueue.h\"\n\nusing moodycamel::BlockingConcurrentQueue;\nusing moodycamel::ConsumerToken;\nusing moodycamel::ProducerToken;\n\n#include \"echo.grpc.pb.h\"\n#include \"echo.pb.h\"\n\n/**\n * @brief Batching Service for Unary Requests\n *\n * Exposes a Unary (send/recv) interface for a given RPC, but rather than\n * computing the RPC, the service simply batches the incoming requests and\n * forwards them via a gRPC stream to a service that implements the actual\n * compute portion of the RPC.\n *\n * The backend compute service is not a Unary service. Rather it must\n * implemented the LifeCycleBatching service Context, i.e. BatchingContext.\n * The other application in this folder implements the backend service.\n *\n * Streams are used as a forwarding mechanism because of how they interact\n * with a load-balancer. Unlike unary requests which get balanced on each\n * request, a stream only get balanced when it is opened. All items of a stream\n * go to the same endpoint service.\n *\n * @tparam ServiceType\n * @tparam Request\n * @tparam Response\n */\ntemplate\nstruct BatchingService\n{\n using Callback = std::function;\n\n struct MessageType\n {\n Request* request;\n Response* response;\n Callback callback;\n };\n\n /**\n * @brief Forwards incoming Unary requests via a gRPC Stream to\n * a Batched Steaming Service that implements the actual RPC\n */\n class Client\n {\n public:\n using PrepareFunc =\n std::function>(\n ::grpc::ClientContext*, ::grpc::CompletionQueue*)>;\n\n Client(PrepareFunc prepare_func, std::shared_ptr thread_pool)\n : m_PrepareFunc(prepare_func), m_ThreadPool(thread_pool), m_CurrentCQ(0)\n {\n for(decltype(m_ThreadPool->Size()) i = 0; i < m_ThreadPool->Size(); i++)\n {\n LOG(INFO) << \"Starting Client Progress Engine #\" << i;\n m_CQs.emplace_back(new ::grpc::CompletionQueue);\n auto cq = m_CQs.back().get();\n m_ThreadPool->enqueue([this, cq] { ProgressEngine(*cq); });\n }\n }\n\n void WriteAndCloseStream(uint32_t messages_count, MessageType* messages)\n {\n auto cq = m_CQs[++m_CurrentCQ % m_CQs.size()].get();\n LOG(INFO) << \"Client using CQ: \" << (void*)cq;\n\n auto ctx = new Call;\n for(uint32_t i = 0; i < messages_count; i++)\n {\n ctx->Push(messages[i]);\n }\n\n ctx->m_Stream = m_PrepareFunc(&ctx->m_Context, cq);\n ctx->Start();\n }\n\n private:\n class Call\n {\n public:\n Call() : m_Started(false), m_NextState(&Call::StateInvalid) {}\n virtual ~Call() {}\n\n void Push(MessageType& message)\n {\n if(m_Started) LOG(FATAL) << \"Stream started; No pushing allowed.\";\n m_Requests.push(message.request);\n m_Responses.push(message.response);\n m_CallbackByResponse[message.response] = message.callback;\n }\n\n void Start()\n {\n LOG(INFO) << \"Starting Batch Forwarding of Size \" << m_Requests.size()\n << \" for Tag \" << Tag();\n m_NextState = &Call::StateWriteDone;\n m_Stream->StartCall(Tag());\n }\n\n private:\n bool RunNextState(bool ok)\n {\n bool ret = (this->*m_NextState)(ok);\n if(!ret) DLOG(INFO) << \"RunNextState returning false\";\n return ret;\n }\n\n void* Tag() { return static_cast(this); }\n\n bool Fail()\n {\n LOG(FATAL) << \"Fail\";\n return false;\n }\n\n void WriteNext()\n {\n if(m_Requests.size())\n {\n auto request = m_Requests.front();\n m_Requests.pop();\n DLOG(INFO) << \"forwarding request\";\n m_NextState = &Call::StateWriteDone;\n m_Stream->Write(*request, Tag());\n }\n else\n {\n DLOG(INFO) << \"closing client stream for writing\";\n m_NextState = &Call::StateCloseStreamDone;\n m_Stream->WritesDone(Tag());\n }\n }\n\n void ReadNext()\n {\n if(m_Responses.size())\n {\n DLOG(INFO) << \"waiting on response\";\n auto response = m_Responses.front();\n m_NextState = &Call::StateReadDone;\n m_Stream->Read(response, Tag());\n }\n else\n {\n DLOG(INFO) << \"waiting on finished message from server\";\n m_NextState = &Call::StateFinishedDone;\n m_Stream->Finish(&m_Status, Tag());\n }\n }\n\n bool StateWriteDone(bool ok)\n {\n if(!ok) return Fail();\n DLOG(INFO) << \"request forwarded!\";\n WriteNext();\n return true;\n }\n\n bool StateReadDone(bool ok)\n {\n if(!ok) return Fail();\n DLOG(INFO) << \"response received\";\n auto response = m_Responses.front();\n m_Responses.pop();\n auto search = m_CallbackByResponse.find(response);\n if(search == m_CallbackByResponse.end())\n LOG(FATAL) << \"Callback for response not found\";\n ReadNext();\n // Execute callback which will complete the unary request for this stream item\n DLOG(INFO) << \"triggering callback on held receive context\";\n search->second(true);\n DLOG(INFO) << \"callback completed\";\n return true;\n }\n\n bool StateCloseStreamDone(bool ok)\n {\n if(!ok) return Fail();\n DLOG(INFO) << \"closed client stream for writing\";\n ReadNext();\n return true;\n }\n\n bool StateFinishedDone(bool ok)\n {\n if(m_Status.ok())\n DLOG(INFO) << \"ClientContext: \" << Tag() << \" finished with OK\";\n else\n DLOG(INFO) << \"ClientContext: \" << Tag() << \" finished with CANCELLED\";\n m_NextState = &Call::StateInvalid;\n LOG(INFO) << \"Batch Forwarding Completed for Tag \" << Tag();\n return false;\n }\n\n bool StateInvalid(bool ok) { LOG(FATAL) << \"This should never be called\"; }\n\n private:\n std::queue m_Requests;\n std::queue m_Responses;\n std::map m_CallbackByResponse;\n\n bool (Call::*m_NextState)(bool);\n\n ::grpc::Status m_Status;\n ::grpc::ClientContext m_Context;\n std::unique_ptr<::grpc::ClientAsyncReaderWriter> m_Stream;\n bool m_Started;\n\n friend class Client;\n };\n\n void ProgressEngine(::grpc::CompletionQueue& cq)\n {\n void* tag;\n bool ok = false;\n\n while(cq.Next(&tag, &ok))\n {\n CHECK(ok) << \"not ok\";\n Call* call = static_cast(tag);\n if(!call->RunNextState(ok))\n {\n DLOG(INFO) << \"Deleting Stream: \" << tag;\n delete call;\n }\n }\n }\n\n int m_CurrentCQ;\n PrepareFunc m_PrepareFunc;\n std::shared_ptr m_ThreadPool;\n std::vector> m_CQs;\n };\n\n class Resources : public ::trtlab::Resources\n {\n public:\n Resources(uint32_t max_batch_size, uint64_t timeout, std::shared_ptr client)\n : m_MaxBatchsize(max_batch_size), m_Timeout(timeout), m_Client(client)\n {\n }\n\n virtual void PreprocessRequest(Request* req) {}\n\n void Push(Request* req, Response* resp, Callback callback)\n {\n // thread_local ProducerToken token(m_MessageQueue);\n // m_MessageQueue.enqueue(token, MessageType(req, resp, callback));\n PreprocessRequest(req);\n m_MessageQueue.enqueue(MessageType{req, resp, callback});\n }\n\n void ProgressEngine()\n {\n constexpr uint64_t quanta = 100;\n const double timeout = static_cast(m_Timeout - quanta) / 1000000.0;\n size_t total_count;\n size_t max_batch;\n std::vector messages;\n messages.resize(m_MaxBatch)\n\n thread_local ConsumerToken token(m_MessageQueue);\n for(;;)\n {\n max_batch = m_MaxBatchsize;\n total_count = 0;\n auto start = std::chrono::steady_clock::now();\n auto elapsed = [start]() -> double {\n return std::chrono::duration(std::chrono::steady_clock::now() - start)\n .count();\n };\n\n // initial pull - if not successful, restart loop\n\n // if successful, then open a stream, push message to stream and continue to collect\n // requests until the max_batch_size is reach for the timeout is triggered\n\n // finish sending\n\n // r\n do\n {\n auto count = m_MessageQueue.wait_dequeue_bulk_timed(\n token, &messages[total_count], max_batch, quanta);\n total_count += count;\n max_batch -= count;\n } while(total_count && total_count < m_MaxBatchsize && elapsed() < timeout);\n if(total_count)\n {\n m_Client->WriteAndCloseStream(total_count, messages);\n }\n }\n }\n\n private:\n size_t m_MaxBatchsize;\n uint64_t m_Timeout;\n std::shared_ptr m_Client;\n BlockingConcurrentQueue m_MessageQueue;\n };\n\n class ReceiveContext final : public ::nvrpc::Context\n {\n void ExecuteRPC(Request& request, Response& response) final override\n {\n LOG(INFO) << \"incoming unary request\";\n this->GetResources()->Push(&request, &response, [this](bool ok) {\n if(ok)\n this->FinishResponse();\n else\n {\n LOG(INFO) << \"shoot\";\n this->CancelResponse();\n }\n });\n }\n };\n};\n\nDEFINE_uint32(max_batch_size, 8, \"Maximum batch size to collect and foward\");\nDEFINE_uint64(timeout_usecs, 2000, \"Batching window timeout in microseconds\");\nDEFINE_uint32(max_batches_in_flight, 1, \"Maximum number of forwarded batches\");\nDEFINE_uint32(receiving_threads, 1, \"Number of Forwarding threads\");\nDEFINE_uint32(forwarding_threads, 1, \"Number of Forwarding threads\");\nDEFINE_string(forwarding_target, \"localhost:50051\", \"Batched Compute Service / Load-Balancer\");\n\nusing InferenceBatchingService = BatchingService;\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"simpleBatchingService\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n auto forwarding_threads = std::make_shared(FLAGS_forwarding_threads);\n auto channel = grpc::CreateChannel(FLAGS_forwarding_target, grpc::InsecureChannelCredentials());\n auto stub = ::simple::Inference::NewStub(channel);\n auto forwarding_prepare_func = [&stub](::grpc::ClientContext * context,\n ::grpc::CompletionQueue * cq) -> auto\n {\n return std::move(stub->PrepareAsyncBatchedCompute(context, cq));\n };\n\n auto client = std::make_shared(forwarding_prepare_func,\n forwarding_threads);\n\n auto rpcResources = std::make_shared(\n FLAGS_max_batch_size, FLAGS_timeout_usecs, client);\n\n Server server(\"0.0.0.0:50049\");\n auto recvService = server.RegisterAsyncService<::simple::Inference>();\n auto rpcCompute = recvService->RegisterRPC(\n &::simple::Inference::AsyncService::RequestCompute);\n\n uint64_t context_count = FLAGS_max_batch_size * FLAGS_max_batches_in_flight;\n uint64_t contexts_per_executor_thread = std::max(context_count / FLAGS_receiving_threads, 1UL);\n\n auto executor = server.RegisterExecutor(new Executor(FLAGS_receiving_threads));\n executor->RegisterContexts(rpcCompute, rpcResources, contexts_per_executor_thread);\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(1), [rpcResources] { rpcResources->ProgressEngine(); });\n\n return 0;\n}\n" }, { "path": "examples/03_Batching/launch_batching.sh", "content": "#!/bin/bash -e\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\ncleanup() {\n kill $(jobs -p) ||:\n}\ntrap \"cleanup\" EXIT SIGINT SIGTERM\n\nsleep 1\n\necho \"starting streaming services\"\n/work/build/examples/03_Batching/streaming-service-echo.x &\nwait-for-it.sh localhost:50051 --timeout=0 -- echo \"Streaming service is ready.\"\n\necho \"starting batching service\"\n/work/build/examples/03_Batching/batching-service-echo.x &\nwait-for-it.sh localhost:50049 --timeout=0 -- echo \"Batching service is ready.\"\n\necho\necho \"Starting a shell keeping the services and load-balancer running...\"\necho \"Try python unary_client.py - exit shell to kill services\"\nbash --rcfile <(echo \"PS1='Batching Subshell: '\")\n" }, { "path": "examples/03_Batching/simple_batching_client.py", "content": "import grpc\n\nimport simple_pb2\nimport simple_pb2_grpc\n\n\ndef run():\n with grpc.insecure_channel('localhost:50051') as channel:\n stub = simple_pb2_grpc.InferenceStub(channel)\n def requests():\n messages = [simple_pb2.Input(batch_id=i) for i in range(10)]\n for msg in messages:\n print(\"Sending Stream batch_id={}\".format(msg.batch_id))\n yield msg\n\n responses = stub.BatchedCompute(requests())\n for resp in responses:\n print(\"Received msg on stream with batch_id={}\".format(resp.batch_id))\n\nif __name__ == \"__main__\":\n run() \n" }, { "path": "examples/03_Batching/simple_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: simple.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\nfrom google.protobuf import descriptor_pb2\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='simple.proto',\n package='simple',\n syntax='proto3',\n serialized_pb=_b('\\n\\x0csimple.proto\\x12\\x06simple\\\"\\x19\\n\\x05Input\\x12\\x10\\n\\x08\\x62\\x61tch_id\\x18\\x01 \\x01(\\x04\\\"\\x1a\\n\\x06Output\\x12\\x10\\n\\x08\\x62\\x61tch_id\\x18\\x01 \\x01(\\x04\\x32n\\n\\tInference\\x12*\\n\\x07\\x43ompute\\x12\\r.simple.Input\\x1a\\x0e.simple.Output\\\"\\x00\\x12\\x35\\n\\x0e\\x42\\x61tchedCompute\\x12\\r.simple.Input\\x1a\\x0e.simple.Output\\\"\\x00(\\x01\\x30\\x01\\x62\\x06proto3')\n)\n\n\n\n\n_INPUT = _descriptor.Descriptor(\n name='Input',\n full_name='simple.Input',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='batch_id', full_name='simple.Input.batch_id', index=0,\n number=1, type=4, cpp_type=4, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=24,\n serialized_end=49,\n)\n\n\n_OUTPUT = _descriptor.Descriptor(\n name='Output',\n full_name='simple.Output',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='batch_id', full_name='simple.Output.batch_id', index=0,\n number=1, type=4, cpp_type=4, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=51,\n serialized_end=77,\n)\n\nDESCRIPTOR.message_types_by_name['Input'] = _INPUT\nDESCRIPTOR.message_types_by_name['Output'] = _OUTPUT\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nInput = _reflection.GeneratedProtocolMessageType('Input', (_message.Message,), dict(\n DESCRIPTOR = _INPUT,\n __module__ = 'simple_pb2'\n # @@protoc_insertion_point(class_scope:simple.Input)\n ))\n_sym_db.RegisterMessage(Input)\n\nOutput = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), dict(\n DESCRIPTOR = _OUTPUT,\n __module__ = 'simple_pb2'\n # @@protoc_insertion_point(class_scope:simple.Output)\n ))\n_sym_db.RegisterMessage(Output)\n\n\n\n_INFERENCE = _descriptor.ServiceDescriptor(\n name='Inference',\n full_name='simple.Inference',\n file=DESCRIPTOR,\n index=0,\n options=None,\n serialized_start=79,\n serialized_end=189,\n methods=[\n _descriptor.MethodDescriptor(\n name='Compute',\n full_name='simple.Inference.Compute',\n index=0,\n containing_service=None,\n input_type=_INPUT,\n output_type=_OUTPUT,\n options=None,\n ),\n _descriptor.MethodDescriptor(\n name='BatchedCompute',\n full_name='simple.Inference.BatchedCompute',\n index=1,\n containing_service=None,\n input_type=_INPUT,\n output_type=_OUTPUT,\n options=None,\n ),\n])\n_sym_db.RegisterServiceDescriptor(_INFERENCE)\n\nDESCRIPTOR.services_by_name['Inference'] = _INFERENCE\n\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "examples/03_Batching/simple_pb2_grpc.py", "content": "# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!\nimport grpc\n\nimport simple_pb2 as simple__pb2\n\n\nclass InferenceStub(object):\n # missing associated documentation comment in .proto file\n pass\n\n def __init__(self, channel):\n \"\"\"Constructor.\n\n Args:\n channel: A grpc.Channel.\n \"\"\"\n self.Compute = channel.unary_unary(\n '/simple.Inference/Compute',\n request_serializer=simple__pb2.Input.SerializeToString,\n response_deserializer=simple__pb2.Output.FromString,\n )\n self.BatchedCompute = channel.stream_stream(\n '/simple.Inference/BatchedCompute',\n request_serializer=simple__pb2.Input.SerializeToString,\n response_deserializer=simple__pb2.Output.FromString,\n )\n\n\nclass InferenceServicer(object):\n # missing associated documentation comment in .proto file\n pass\n\n def Compute(self, request, context):\n # missing associated documentation comment in .proto file\n pass\n context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n context.set_details('Method not implemented!')\n raise NotImplementedError('Method not implemented!')\n\n def BatchedCompute(self, request_iterator, context):\n # missing associated documentation comment in .proto file\n pass\n context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n context.set_details('Method not implemented!')\n raise NotImplementedError('Method not implemented!')\n\n\ndef add_InferenceServicer_to_server(servicer, server):\n rpc_method_handlers = {\n 'Compute': grpc.unary_unary_rpc_method_handler(\n servicer.Compute,\n request_deserializer=simple__pb2.Input.FromString,\n response_serializer=simple__pb2.Output.SerializeToString,\n ),\n 'BatchedCompute': grpc.stream_stream_rpc_method_handler(\n servicer.BatchedCompute,\n request_deserializer=simple__pb2.Input.FromString,\n response_serializer=simple__pb2.Output.SerializeToString,\n ),\n }\n generic_handler = grpc.method_handlers_generic_handler(\n 'simple.Inference', rpc_method_handlers)\n server.add_generic_rpc_handlers((generic_handler,))\n" }, { "path": "examples/03_Batching/streaming-service.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \"nvrpc/context.h\"\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/rpc.h\"\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n#include \"tensorrt/laboratory/core/resources.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n\nusing nvrpc::AsyncRPC;\nusing nvrpc::AsyncService;\nusing nvrpc::BatchingContext;\nusing nvrpc::Executor;\nusing nvrpc::Server;\nusing trtlab::Resources;\nusing trtlab::ThreadPool;\n\n#include \"echo.grpc.pb.h\"\n#include \"echo.pb.h\"\n\nclass SimpleContext final : public BatchingContext\n{\n void ExecuteRPC(std::vector& inputs,\n std::vector& outputs) final override\n {\n for(auto input = inputs.cbegin(); input != inputs.cend(); input++)\n {\n auto output = outputs.emplace(outputs.end());\n output->set_batch_id(input->batch_id());\n LOG(INFO) << \"Response with batch_id=\" << output->batch_id();\n }\n this->FinishResponse();\n }\n\n void OnRequestReceived(const RequestType& request) final override\n {\n LOG(INFO) << \"Recieved request with batch_id=\" << request.batch_id();\n }\n};\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n\n ::google::InitGoogleLogging(\"simpleServer\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n Server server(\"0.0.0.0:50051\");\n\n LOG(INFO) << \"Register Service (simple::Inference)\";\n auto simpleInference = server.RegisterAsyncService();\n\n LOG(INFO)\n << \"Register RPC (simple::Inference::BatchedCompute) with Service (simple::Inference)\";\n auto rpcCompute = simpleInference->RegisterRPC(\n &simple::Inference::AsyncService::RequestBatchedCompute);\n\n LOG(INFO) << \"Initializing Resources for RPC (simple::Inference::BatchedCompute)\";\n auto rpcResources = std::make_shared();\n\n LOG(INFO) << \"Creating Executor\";\n auto executor = server.RegisterExecutor(new Executor(1));\n\n LOG(INFO) << \"Creating Execution Contexts for RPC (simple::Inference::Compute) with Executor\";\n executor->RegisterContexts(rpcCompute, rpcResources, 10);\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(2000), [] {\n // This is a timeout loop executed every 2seconds\n // Run() with no arguments will run an empty timeout loop every 5 seconds.\n // RunAsync() will return immediately, its your responsibility to ensure the\n // server doesn't go out of scope or a Shutdown will be triggered on your services.\n });\n}\n" }, { "path": "examples/03_Batching/unary_client.py", "content": "import grpc\n\nimport simple_pb2\nimport simple_pb2_grpc\n\n\ndef run():\n with grpc.insecure_channel('localhost:50049') as channel:\n stub = simple_pb2_grpc.InferenceStub(channel)\n \n response = stub.Compute(simple_pb2.Input(batch_id=78))\n print(\"Received msg with batch_id={}\".format(response.batch_id))\n\nif __name__ == \"__main__\":\n run() \n" }, { "path": "examples/04_Middleman/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nadd_executable(middleman-client.x\n middleman-client.cc\n)\n\ntarget_link_libraries(middleman-client.x\n trtlab::nvrpc\n nv-inference-protos\n gflags\n)\n" }, { "path": "examples/04_Middleman/middleman-client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \"tensorrt/laboratory/core/memory/allocator.h\"\n#include \"tensorrt/laboratory/core/memory/malloc.h\"\n\nusing trtlab::Allocator;\nusing trtlab::Malloc;\n\n#include \"nvrpc/context.h\"\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n\nusing nvrpc::Context;\nusing nvrpc::Executor;\nusing nvrpc::Server;\nusing trtlab::ThreadPool;\n\n#include \"moodycamel/blockingconcurrentqueue.h\"\n\nusing moodycamel::BlockingConcurrentQueue;\nusing moodycamel::ConsumerToken;\nusing moodycamel::ProducerToken;\n\n// NVIDIA Inference Server Protos\n#include \"nvidia_inference.grpc.pb.h\"\n#include \"nvidia_inference.pb.h\"\n\nnamespace easter = ::nvidia::inferenceserver;\n/*\nusing nvidia::inferenceserver::GRPCService;\nusing nvidia::inferenceserver::InferRequest;\nusing nvidia::inferenceserver::InferResponse;\n*/\n\n/**\n * @brief Batching Service for Unary Requests\n *\n * Exposes a Unary (send/recv) interface for a given RPC, but rather than\n * computing the RPC, the service simply batches the incoming requests and\n * forwards them via a gRPC stream to a service that implements the actual\n * compute portion of the RPC.\n *\n * The backend compute service is not a Unary service. Rather it must\n * implemented the LifeCycleBatching service Context, i.e. BatchingContext.\n * The other application in this folder implements the backend service.\n *\n * Streams are used as a forwarding mechanism because of how they interact\n * with a load-balancer. Unlike unary requests which get balanced on each\n * request, a stream only get balanced when it is opened. All items of a stream\n * go to the same endpoint service.\n *\n * @tparam ServiceType\n * @tparam Request\n * @tparam Response\n */\ntemplate\nstruct MiddlemanService\n{\n using Callback = std::function;\n\n struct MessageType\n {\n Request* request;\n Response* response;\n Callback callback;\n };\n\n /**\n * @brief Forwards incoming Unary requests via a gRPC Stream to\n * a Batched Steaming Service that implements the actual RPC\n */\n class Client\n {\n public:\n using PrepareFunc =\n std::function>(\n ::grpc::ClientContext*, const Request&, ::grpc::CompletionQueue*)>;\n\n Client(PrepareFunc prepare_func, std::shared_ptr thread_pool)\n : m_PrepareFunc(prepare_func), m_ThreadPool(thread_pool), m_CurrentCQ(0)\n {\n for(decltype(m_ThreadPool->Size()) i = 0; i < m_ThreadPool->Size(); i++)\n {\n LOG(INFO) << \"Starting Client Progress Engine #\" << i;\n m_CQs.emplace_back(new ::grpc::CompletionQueue);\n auto cq = m_CQs.back().get();\n m_ThreadPool->enqueue([this, cq] { ProgressEngine(*cq); });\n }\n }\n\n void WriteAndCloseStream(uint32_t messages_count, MessageType* messages)\n {\n auto cq = m_CQs[++m_CurrentCQ % m_CQs.size()].get();\n DLOG(INFO) << \"Client using CQ: \" << (void*)cq;\n CHECK_EQ(1U, messages_count) << \"forwarder; not batcher\";\n\n auto ctx = new Call;\n for(uint32_t i = 0; i < messages_count; i++)\n {\n ctx->Push(messages[i]);\n }\n\n ctx->m_Reader = m_PrepareFunc(&ctx->m_Context, *ctx->m_Request, cq);\n ctx->m_Reader->StartCall();\n ctx->m_Reader->Finish(ctx->m_Response, &ctx->m_Status, ctx->Tag());\n }\n\n private:\n class Call\n {\n public:\n Call() : m_NextState(&Call::StateFinishedDone) {}\n virtual ~Call() {}\n\n void Push(MessageType& message)\n {\n m_Request = message.request;\n m_Response = message.response;\n m_Callback = message.callback;\n }\n\n private:\n bool RunNextState(bool ok)\n {\n bool ret = (this->*m_NextState)(ok);\n if(!ret) DLOG(INFO) << \"RunNextState returning false\";\n return ret;\n }\n\n void* Tag() { return static_cast(this); }\n\n bool Fail()\n {\n LOG(FATAL) << \"Fail\";\n return false;\n }\n\n bool StateFinishedDone(bool ok)\n {\n if(m_Status.ok())\n DLOG(INFO) << \"ClientContext: \" << Tag() << \" finished with OK\";\n else\n DLOG(INFO) << \"ClientContext: \" << Tag() << \" finished with CANCELLED\";\n m_Callback(m_Status.ok());\n DLOG(INFO) << \"Forwarding Completed for Tag \" << Tag();\n return false;\n }\n\n private:\n Request* m_Request;\n Response* m_Response;\n Callback m_Callback;\n\n bool (Call::*m_NextState)(bool);\n\n ::grpc::Status m_Status;\n ::grpc::ClientContext m_Context;\n std::unique_ptr<::grpc::ClientAsyncResponseReader> m_Reader;\n\n friend class Client;\n };\n\n void ProgressEngine(::grpc::CompletionQueue& cq)\n {\n void* tag;\n bool ok = false;\n\n while(cq.Next(&tag, &ok))\n {\n CHECK(ok) << \"not ok\";\n Call* call = static_cast(tag);\n if(!call->RunNextState(ok))\n {\n DLOG(INFO) << \"Deleting Stream: \" << tag;\n delete call;\n }\n }\n }\n\n int m_CurrentCQ;\n PrepareFunc m_PrepareFunc;\n std::shared_ptr m_ThreadPool;\n std::vector> m_CQs;\n };\n\n public:\n class Resources : public ::trtlab::Resources\n {\n public:\n Resources(uint32_t max_batch_size, uint64_t timeout, std::shared_ptr client)\n : m_MaxBatchsize(max_batch_size), m_Timeout(timeout), m_Client(client)\n {\n }\n\n virtual void PreprocessRequest(Request* req) {}\n\n void Push(Request* req, Response* resp, Callback callback)\n {\n // thread_local ProducerToken token(m_MessageQueue);\n // m_MessageQueue.enqueue(token, MessageType(req, resp, callback));\n PreprocessRequest(req);\n m_MessageQueue.enqueue(MessageType{req, resp, callback});\n }\n\n void ProgressEngine()\n {\n constexpr uint64_t quanta = 100;\n const double timeout = static_cast(m_Timeout - quanta) / 1000000.0;\n size_t total_count;\n size_t max_batch;\n\n thread_local ConsumerToken token(m_MessageQueue);\n for(;;)\n {\n MessageType messages[m_MaxBatchsize];\n max_batch = m_MaxBatchsize;\n total_count = 0;\n auto start = std::chrono::steady_clock::now();\n auto elapsed = [start]() -> double {\n return std::chrono::duration(std::chrono::steady_clock::now() - start)\n .count();\n };\n do\n {\n auto count = m_MessageQueue.wait_dequeue_bulk_timed(\n token, &messages[total_count], max_batch, quanta);\n CHECK_LE(count, max_batch);\n total_count += count;\n max_batch -= count;\n } while(total_count && total_count < m_MaxBatchsize && elapsed() < timeout);\n if(total_count)\n {\n m_Client->WriteAndCloseStream(total_count, messages);\n }\n }\n }\n\n private:\n size_t m_MaxBatchsize;\n uint64_t m_Timeout;\n std::shared_ptr m_Client;\n BlockingConcurrentQueue m_MessageQueue;\n };\n\n class ReceiveContext final : public ::nvrpc::Context\n {\n void ExecuteRPC(Request& request, Response& response) final override\n {\n DLOG(INFO) << \"incoming unary request\";\n this->GetResources()->Push(&request, &response, [this](bool ok) {\n if(ok)\n this->FinishResponse();\n else\n {\n LOG(INFO) << \"shoot\";\n this->CancelResponse();\n }\n });\n }\n };\n};\n\nDEFINE_uint32(max_batch_size, 1, \"Maximum batch size to collect and foward\");\nDEFINE_uint64(timeout_usecs, 200, \"Batching window timeout in microseconds\");\nDEFINE_uint32(max_batches_in_flight, 300, \"Maximum number of forwarded batches\");\nDEFINE_uint32(receiving_threads, 2, \"Number of Forwarding threads\");\nDEFINE_uint32(forwarding_threads, 2, \"Number of Forwarding threads\");\nDEFINE_string(forwarding_target, \"localhost:8001\", \"Batched Compute Service / Load-Balancer\");\n\nusing InferMiddlemanService =\n MiddlemanService;\nusing StatusMiddlemanService =\n MiddlemanService;\n\nclass DemoMiddlemanService : public InferMiddlemanService\n{\n public:\n class Resources : public InferMiddlemanService::Resources\n {\n public:\n using InferMiddlemanService::Resources::Resources;\n void PreprocessRequest(easter::InferRequest* req) override\n {\n static auto local_data = std::make_unique>(10 * 1024 * 1024);\n DLOG(INFO) << \"Boom - preprocess request here!\";\n auto bytes = req->meta_data().batch_size() * req->meta_data().input(0).byte_size();\n CHECK_EQ(0, req->raw_input_size());\n req->add_raw_input(local_data->Data(), bytes);\n }\n };\n};\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"easterForwardingService\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n grpc::ChannelArguments ch_args;\n ch_args.SetMaxReceiveMessageSize(-1);\n auto channel = grpc::CreateCustomChannel(FLAGS_forwarding_target,\n grpc::InsecureChannelCredentials(), ch_args);\n\n // GRPCService::Infer async forwarder\n auto forwarding_threads = std::make_shared(FLAGS_forwarding_threads);\n auto stub = ::easter::GRPCService::NewStub(channel);\n auto forwarding_prepare_func = [&stub](::grpc::ClientContext * context,\n const ::easter::InferRequest& request,\n ::grpc::CompletionQueue* cq) -> auto\n {\n return std::move(stub->PrepareAsyncInfer(context, request, cq));\n };\n auto client =\n std::make_shared(forwarding_prepare_func, forwarding_threads);\n\n // GRPCService::Status async forwarder\n auto status_forwarding_threads = std::make_shared(1);\n auto status_stub = ::easter::GRPCService::NewStub(channel);\n auto status_forwarding_prepare_func = [&stub](::grpc::ClientContext * context,\n const ::easter::StatusRequest& request,\n ::grpc::CompletionQueue* cq) -> auto\n {\n return std::move(stub->PrepareAsyncStatus(context, request, cq));\n };\n auto status_client = std::make_shared(\n status_forwarding_prepare_func, status_forwarding_threads);\n\n auto rpcResources = std::make_shared(\n FLAGS_max_batch_size, FLAGS_timeout_usecs, client);\n\n auto statusResources = std::make_shared(\n FLAGS_max_batch_size, FLAGS_timeout_usecs, status_client);\n\n Server server(\"0.0.0.0:50049\");\n auto bytes = trtlab::StringToBytes(\"100MiB\");\n server.Builder().SetMaxReceiveMessageSize(bytes);\n LOG(INFO) << \"gRPC MaxReceiveMessageSize = \" << trtlab::BytesToString(bytes);\n\n auto recvService = server.RegisterAsyncService<::easter::GRPCService>();\n auto rpcCompute = recvService->RegisterRPC(\n &::easter::GRPCService::AsyncService::RequestInfer);\n auto rpcStatus = recvService->RegisterRPC(\n &::easter::GRPCService::AsyncService::RequestStatus);\n\n uint64_t context_count = FLAGS_max_batch_size * FLAGS_max_batches_in_flight;\n uint64_t contexts_per_executor_thread = std::max(context_count / FLAGS_receiving_threads, 1UL);\n\n auto executor = server.RegisterExecutor(new Executor(FLAGS_receiving_threads));\n executor->RegisterContexts(rpcCompute, rpcResources, contexts_per_executor_thread);\n\n auto status_executor = server.RegisterExecutor(new Executor(1));\n status_executor->RegisterContexts(rpcStatus, statusResources, 1);\n\n auto executor_threads = std::make_shared(2);\n executor_threads->enqueue([rpcResources] { rpcResources->ProgressEngine(); });\n executor_threads->enqueue([statusResources] { statusResources->ProgressEngine(); });\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(1), [] {});\n}\n" }, { "path": "examples/10_Internals/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nadd_executable(internals.x\n internals.cc\n)\ntarget_link_libraries(internals.x\n trtlab::core\n trtlab::cuda\n gflags\n)" }, { "path": "examples/10_Internals/README.md", "content": "# YAIS Internals\n\nThe `internals.x` program is designed to be run on a DGX-Station or DGX-1. This is mostly to highlight\nthe use of the Affinity API. If you want to run on a different CPU architecture, you simply need to\nchange the following lines to a range that works with your CPU.\n\n```\n // Socket 1 - non-hyperthreads on a DGX-1, or\n // Socket 0 - hyperthreads on a DGX-Station\n auto socket_1 = Affinity::GetAffinity().Intersection(\n Affinity::GetCpusFromString(\"20-39\") // <== Change Me!\n );\n```\n\n## Primative Classes\n\n * `Affinity` \n * Get and Set CPU affinities for current thread\n * `ThreadPool` \n * Create generic worker thread pool that accept arbiturary lambda functions.\n * Pinned threads from `ThreadPools` are used to allocate memory to ensure that the CPU allocation\n are allocated and first-touched on the threads on the NUMA node for which it will be used.\n This is important for keeping threads and memory pool separate on NUMA systems.\n * `Memory`\n * `Memory` and the derived classes (`MallocMemory`, `CudaPinnedHostMemory`, `CudaDeviceMemory`) are not\n used directly; however they provided the implmentation details used by the generic `Allocator`.\n * `Allocator`\n * Generic Templated Class used to create `std::shared_ptr` and `std::unique_ptr` to instances of\n `Allocator`.\n * `MemoryStack`\n * Generic Templated Class to create a memory stack from a given `AllocatorType`.\n * You can only advance the stack pointer, or reset the entire stack.\n * TODO: Create sub-stacks from a given stack.\n * `Pool`\n * Generic Templated Class that holds objects of `ResourceType`.\n * Resources can be checked-out of the Pool (Pop) as a *special-type* of `std::shared_ptr`,\n which automatically returned the Resource object the pool when the reference count of the\n `shared_ptr` goes to zero. This Resources are not lost on exceptions, but also that the Pool can not\n be deleted until all object have been returned to the Pool.\n\n## TensorRT Classes\n\n * `Model` \n * Wrapper around `nvinfer1::ICudaEngine`\n * `Buffers` \n * `MemoryStackWithTracking` and `MemoryStackWithTracking` used\n to manage Input/Output Tensor Bindings.\n * Owns a `cudaStream_t` to be used with Async Copies and Kernel Executions on the data held by the Buffers.\n * Convenience H2D and D2H copy functions\n * `ExecutionContext` - Wrapper around `nvinfer1::IExecutionContext`\n * `Enqueue` launches the inference calculuation and adds a `cudaEvent_t` to the stream to be triggered\n when the inference calcuation is finished and the `ExecutionContext` can be released.\n * `Resources`\n * Combines the above set of resources into a single `trtlab::Resources` class capable of being associated\n with a `nvrpc::Context`.\n\n\n## Examples\n\n### Affinity\n\n * [Definition: tensorrt/laboratory/core/affinity.h](../../yais/include/tensorrt/laboratory/core/affinity.h)\n * [Implementation: YAIS/Affinity.cc](../../yais/src/Affinity.cc)\n\nIn this, we request the all logical CPUs from Socket 0 that are not hyperthreads, then we get either all \nthe non-hyperthreads from socket_1 on a DGX-1, or the hyperthreads on socket0 on a DGX-Station using \n`GetCpusFromString`.\n\n```\n // Socket 0 - non-hyperthreads on a DGX-1 or Station\n auto socket_0 = Affinity::GetAffinity().Intersection(\n Affinity::GetCpusBySocket(0).Intersection(\n Affinity::GetCpusByProcessingUnit(0)\n ));\n\n // Socket 1 - non-hyperthreads on a DGX-1, or\n // Socket 0 - hyperthreads on a DGX-Station\n auto socket_1 = Affinity::GetAffinity().Intersection(\n Affinity::GetCpusFromString(\"20-39\")\n );\n\n LOG(INFO) << socket_0;\n```\n\nSingle line output reformatted to per-line-indented output for readability.\n```\n0515 07:14:48.007148 10919 test_affinity.cc:61] \n [id: 0, numa: 0, socket: 0, core: 0, processing_unit: 0], \n [id: 1, numa: 0, socket: 0, core: 1, processing_unit: 0], \n [id: 2, numa: 0, socket: 0, core: 2, processing_unit: 0],\n ... omitted for brevity ...\n [id: 18, numa: 0, socket: 0, core: 18, processing_unit: 0], \n [id: 19, numa: 0, socket: 0, core: 19, processing_unit: 0]\n```\n\n### ThreadPool\n\n * [Definition: tensorrt/laboratory/core/thread_pool.h](../../yais/include/tensorrt/laboratory/core/thread_pool.h)\n * [Implementation: YAIS/ThreadPool.cc](../../yais/src/ThreadPool.cc)\n\nThe ThreadPool class creates a pool of worker threads that pull work from a queue. The work queue can\nbe any set of captured lambda functions or function pointers passed to the `enqueue` function.\n\n```\n // Create a ThreadPool where each thread is pinned to one logical CPU in the CpuSet\n auto workers_0 = std::make_shared(socket_0);\n auto workers_1 = std::make_shared(socket_1);\n\n // Create a massive set of threads that can run anywhere our current process is allowed to run\n auto bftp = std::make_unique(128, Affinity::GetAffinity());\n\n // Shutdown the BFTP\n bftp.reset();\n\n // Enqueue some basic logging\n for(int i=0; i<10; i++) {\n auto result = workers->enqueue([i]{\n LOG(INFO) << i << \" \" << Affinity::GetAffinity();\n std::this_thread::sleep_for(std::chrono::milliseconds(10));\n });\n }\n```\n\nAs these ThreadPools are generic, we can enqueue any type of work to them. Many thanks to the original\nauthors Jakob Progsch and Václav Zeman for this incredible useful class. For details on the original\nwork and the modificiations made in this project, see [CREDITS.md](../../CREDITS.md) and the source code.\n\n\n### Memory\n\nOne of the reasons why `Affinity` and `ThreadPool` were introduced prior to `Memory` is that memory on\nNUMA system can be difficult to do correctly. For memory segments that will be primarly used by sets\nof threads, it is very important to first set the affinity of the threads, then allocate and touch each\npage in the memory allocation (first-touch) on the thread that will primarly use the segment. NERSC \nhas a nice [write-up on memory affinity and first touch policies](http://www.nersc.gov/users/computational-systems/cori/application-porting-and-performance/improving-openmp-scaling/).\nIn this section, we'll show how to properly use the `Memory` and `Allocator` classes in a NUMA friendly\nway using `ThreadPool`s.\n\n * [Definition: tensorrt/laboratory/core/memory.h](../../yais/include/tensorrt/laboratory/core/memory.h)\n\nThe `Memory` class and its derived classes, see below, are the core memory classes in YAIS; however,\nthese classes are not direclty used. Instead, they provide the implmentation details on how memory of\ntheir respective classes is to be allocated, freed, and page-aligned. For details, see the comments\nin the source code.\n\nDerived `Memory` Classes:\n * `Malloc`\n * `CudaPinnedHostMemory`\n * `CudaDeviceMemory`\n * `CudaManagedMemory`\n\n### Allocator\n\n * [Definition: tensorrt/laboratory/core/memory.h](../../yais/include/tensorrt/laboratory/core/memory.h)\n\nThe templated `Allocator` class performs memory allocations and freeing operations. This\nclass does not have a public constructor, instead, you are required to use either the `make_shared`\nor `make_unique` static methods. In doing so, the method to free the allocation is captured by the\ndeconstructor which is triggered by the default deleter of `shared_ptr` and `unique_ptr`.\n\nAn allocated memory segments is of type `Allocator` which inherits from `MemoryType`.\nThe base `Memory` class provides three functions, `GetPointer()`, `GetSize()`, and `WriteZeros()`.\n\n```\n std::shared_ptr pinned_0, pinned_1;\n\n auto future_0 = workers_0->enqueue([&pinned_0]{\n pinned_0 = Allocator::make_shared(1024*1024*1024);\n pinned_0->WriteZeros();\n });\n\n auto future_1 = workers_1->enqueue([&pinned_1]{\n pinned_1 = Allocator::make_shared(1024*1024*1024);\n pinned_1->WriteZeros();\n });\n\n future_0.get();\n CHECK(pinned_0) << \"pinned_0 got deallocated - fail\";\n LOG(INFO) << \"pinned_0 (ptr, size): (\" \n << pinned_0->GetPointer() << \", \"\n << pinned_0->GetSize() << \")\";\n```\n\n```\nI0515 08:36:56.619297 13260 test_affinity.cc:59] pinned_0 (ptr, size): (0x1005e000000, 1073741824)\n```\n\n### MemoryStack\n\n * [Definition: tensorrt/laboratory/core/memory_stack.h](../../yais/include/tensorrt/laboratory/core/memory_stack.h)\n\nGeneric `MemoryStack` that takes an `AllocatorType`. The memory stack advances the stack pointer\nvia `Allocate` and resets the stack pointer via `ResetAllocations`. `MemoryStackWithTracking`\nis a specialized derivation that records the pointer and size of each call to `Allocate`.\n`MemoryStackWithTracking` is used in the provided TensorRT classes as a means to push the\ninput/output tensor bindings onto the stack.\n\n```\n std::shared_ptr> gpu_stack_on_socket0;\n\n future_0 = workers_0->enqueue([&gpu_stack_on_socket0]{\n CHECK_EQ(cudaSetDevice(0), CUDA_SUCCESS) << \"Set Device 0 failed\";\n gpu_stack_on_socket0 = std::make_shared<\n MemoryStackWithTracking>(1024*1024*1024);\n });\n\n future_0.get(); // thread allocating gpu_stack_on_socket0 finished with task\n \n LOG(INFO) << \"Push Binding 0 - 10MB - stack_ptr = \" \n << gpu_stack_on_socket0->Allocate(10*1024*1024);\n LOG(INFO) << \"Push Binding 1 - 128MB - stack_ptr = \" \n << gpu_stack_on_socket0->Allocate(128*1024*1024);\n gpu_stack_on_socket0->ResetAllocations();\n```\n\n```\nI0515 09:46:55.159700 14176 test_affinity.cc:78] Push Binding 0 - 10MB - stack_ptr = 0x1009e000000\nI0515 09:46:55.159710 14176 test_affinity.cc:80] Push Binding 1 - 128MB - stack_ptr = 0x1009ea00000\n```\n\n### Pool\n\n * [Definition: tensorrt/laboratory/core/pool.h](../../yais/include/tensorrt/laboratory/core/pool.h)\n\nA `Pool` is a generic of `Queue>` with a special `Pop`\nmethod. The class inherits from `std::enabled_shared_from_this` meaning it must be constructed using\nthe factory method, which ensures the object is owned by a `std::shared_ptr`.\n\nThe `Pop` method of `Pool` is probably the coolest and most contensious component of this\nlibrary. `Pop` pulls an resource off the queue (`from_queue`); however, it does not return this resource.\nInstead, a *new type* of `std::shared_ptr` is created using the raw pointer from `from_pool`.\nThe reason they this is a *new type* of `shared_ptr` is because we provide a custom `Deleter` method that\ncaptures by value (increments reference count) of both `from_pool` and a `shared_ptr` to the pool itself.\n\nThe custom `Deleter` does not free the resource when its reference count goes to zero; rather, it returns\nthe original `from_pool` `shared_ptr` to the pool.\n\nBy capturing a `shared_ptr` to the pool in the `Deleter`, we ensure the the pool can not be freed while\nresources are checkedout. This also ensures that the `shared_ptr` returned from `Pop` is exception\nsafe; meaning, the resource will be returned to the pool if an exception is thrown and caught - it won't\nleak resources.\n\nAlternatively, `Pop` can be called with an `onReturn` lambda function, which will be executed just prior\nto the original object being returned to the Pool. If the `ResourceType` is stateful, this is a good \nchance to clear the state and prepare it for the next use.\n\n```\n struct Buffer\n {\n Buffer(\n std::shared_ptr pinned_,\n std::shared_ptr> gpu_stack_,\n std::shared_ptr workers_\n ) : pinned(pinned_), gpu_stack(gpu_stack_), workers(workers_) {}\n\n // a real example probably includes a deviceID and a stream as part of the buffer\n\n std::shared_ptr pinned;\n std::shared_ptr> gpu_stack;\n std::shared_ptr workers;\n };\n\n auto buffers = Pool::Create();\n\n buffers->EmplacePush(new Buffer(pinned_0, gpu_stack_on_socket0, workers_0));\n buffers->EmplacePush(new Buffer(pinned_1, gpu_stack_on_socket1, workers_1));\n\n for(int i=0; i<6; i++)\n {\n auto buffer = buffers->Pop();\n buffer->workers->enqueue([buffer]{\n // perform some work - regardless of which buffer you got, you are working\n // on a thread properly assocated with the resources\n // note: buffer is captures by value, incrementing its reference count,\n // meaning you have access to it here and when it goes out of scope, it will\n // be returned to the Pool.\n LOG(INFO) << Affinity::GetAffinity();\n });\n }\n```\n\n## TensorRT Examples\n\n * [Definition: YAIS/YAIS/TensorRT/TensorRT.h](../../yais/include/YAIS/TensorRT/TensorRT.h)\n * [Implemenation: YAIS/TensorRT.cc](../../yais/src/TensorRT.cc)\n\nTensoRT classes build on the primatives above. For now, see the comments in the header file, as\nthe header file is pretty well documented.\n" }, { "path": "examples/10_Internals/internals.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \"tensorrt/laboratory/core/affinity.h\"\n#include \"tensorrt/laboratory/core/memory/allocator.h\"\n#include \"tensorrt/laboratory/core/memory/memory_stack.h\"\n#include \"tensorrt/laboratory/core/pool.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n#include \"tensorrt/laboratory/cuda/device_info.h\"\n#include \"tensorrt/laboratory/cuda/memory/cuda_device.h\"\n#include \"tensorrt/laboratory/cuda/memory/cuda_managed.h\"\n#include \"tensorrt/laboratory/cuda/memory/cuda_pinned_host.h\"\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n\nusing trtlab::Affinity;\nusing trtlab::Allocator;\nusing trtlab::CpuSet;\nusing trtlab::CudaDeviceMemory;\nusing trtlab::CudaPinnedHostMemory;\nusing trtlab::DeviceInfo;\nusing trtlab::MemoryStack;\nusing trtlab::Pool;\nusing trtlab::ThreadPool;\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"example10/internals.x\");\n\n auto one_gib = 1024 * 1024 * 1024;\n auto zeroMemory = true;\n\n const auto& gpu_0 = DeviceInfo::Affinity(0);\n\n // Socket 0 - non-hyperthreads on a DGX-1 or Station\n const auto& socket_0 = Affinity::GetAffinity().Intersection(\n Affinity::GetCpusBySocket(0).Intersection(Affinity::GetCpusByProcessingUnit(0)));\n\n // Socket 1 - non-hyperthreads on a DGX-1, or\n // Socket 0 - hyperthreads on a DGX-Station\n const auto& socket_1 =\n Affinity::GetAffinity().Intersection(Affinity::GetCpusFromString(\"20-39\"));\n\n auto workers_0 = std::make_shared(socket_0);\n auto workers_1 = std::make_shared(socket_1);\n\n std::shared_ptr pinned_0, pinned_1;\n\n auto future_0 = workers_0->enqueue([=, &pinned_0] {\n pinned_0 = std::make_shared>(one_gib);\n pinned_0->Fill(0);\n });\n\n auto future_1 = workers_1->enqueue([=, &pinned_1] {\n pinned_1 = std::make_shared>(one_gib);\n pinned_1->Fill(0);\n });\n\n LOG(INFO) << socket_0;\n\n future_0.get();\n CHECK(pinned_0) << \"pinned_0 got deAllocator - fail\";\n LOG(INFO) << \"pinned_0 (ptr, size): (\" << pinned_0->Data() << \", \" << pinned_0->Size() << \")\";\n future_1.get();\n\n std::shared_ptr> gpu_stack_on_socket0;\n std::shared_ptr> gpu_stack_on_socket1;\n\n // It's not strictly necessary to alloaction GPU memory from threads near the GPU\n // this just drives home the point that we want to align CPU worker thread to GPU affinity.\n future_0 = workers_0->enqueue([=, &gpu_stack_on_socket0] {\n CHECK_EQ(cudaSetDevice(0), CUDA_SUCCESS) << \"Set Device 0 failed\";\n gpu_stack_on_socket0 = std::make_shared>(one_gib);\n gpu_stack_on_socket0->Reset(zeroMemory);\n });\n\n // On a dual-socket system, we could use workers_1 to allocation device memory.\n // Leaving this as an exercise to the reader.\n\n future_0.get(); // thread allocating gpu_stack_on_socket0 finished with task\n LOG(INFO) << \"Push Binding 0 - 10MB - stack_ptr = \"\n << gpu_stack_on_socket0->Allocate(10 * 1024 * 1024);\n LOG(INFO) << \"Push Binding 1 - 128MB - stack_ptr = \"\n << gpu_stack_on_socket0->Allocate(128 * 1024 * 1024);\n // Try allocating 1 byte. Notice how the memory is aligned. Default alignment\n // is defined by the MemoryType in Memory.h\n gpu_stack_on_socket0->Reset();\n\n /**\n * Create a Buffer object associates a worker threads, host memory and device memory\n * that are properly aligned to the hardware topology.\n */\n struct Buffer\n {\n Buffer(std::shared_ptr pinned_,\n std::shared_ptr> gpu_stack_,\n std::shared_ptr workers_)\n : pinned(pinned_), gpu_stack(gpu_stack_), workers(workers_)\n {\n }\n\n std::shared_ptr pinned;\n std::shared_ptr> gpu_stack;\n std::shared_ptr workers;\n\n // Normally, we'd associate some GPU index value to the buffer.\n };\n\n // Now create a Pool of Buffers\n auto buffers = Pool::Create();\n\n // Here we push two buffers, one for each socket.\n buffers->EmplacePush(new Buffer(pinned_0, gpu_stack_on_socket0, workers_0));\n buffers->EmplacePush(new Buffer(pinned_1, gpu_stack_on_socket1, workers_1));\n\n // Exercise: add more buffer objects. Which of the three objects per Buffer\n // will you reuse, which will you make new instances of?\n\n // If you have arbituray work which is not necesasry topology aligned, say an incoming\n // inference request, you can pull a buffer object from the pool and queue work to the\n // proper set of threads best associated with that device\n for(int i = 0; i < 6; i++)\n {\n auto buffer = buffers->Pop();\n buffer->workers->enqueue([buffer] {\n // perform some work - regardless of which buffer you got, you are working\n // on a thread properly assocated with the resources\n LOG(INFO) << Affinity::GetAffinity();\n std::this_thread::sleep_for(std::chrono::milliseconds(1));\n });\n }\n\n workers_0.reset();\n workers_1.reset();\n\n return 0;\n}\n" }, { "path": "examples/11_Protos/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n\nset(protobuf_MODULE_COMPATIBLE TRUE)\nfind_package(Protobuf CONFIG REQUIRED)\nmessage(STATUS \"Using protobuf ${protobuf_VERSION}\")\nset(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)\nset(_PROTOBUF_PROTOC $)\n\nadd_subdirectory(echo)\nadd_subdirectory(demo)\nadd_subdirectory(inference)\n" }, { "path": "examples/11_Protos/demo/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nINCLUDE(GRPCGenerateCPP)\n\nPROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS \n dataset.proto\n inference.proto\n)\n\nPROTOBUF_GENERATE_GRPC_CPP(PROTO_GRPC_SRCS PROTO_GRPC_HDRS \n dataset.proto\n inference.proto\n)\n\nadd_library(demo-protos\n ${PROTO_SRCS}\n ${PROTO_GRPC_SRCS}\n)\n\ntarget_link_libraries(demo-protos\n PUBLIC\n ${_PROTOBUF_LIBPROTOBUF}\n)\n\ntarget_include_directories(demo-protos PUBLIC\n ${CMAKE_CURRENT_BINARY_DIR}\n)\n\n" }, { "path": "examples/11_Protos/demo/dataset.proto", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n\nsyntax = \"proto3\";\n\nservice SharedMemoryDataSet {\n rpc GetInfo (InfoRequest) returns (Info) {}\n}\n\nmessage Image {\n fixed64 sysv_offset = 1;\n uint32 label_index = 2;\n repeated int32 shape = 3;\n uint64 size = 4;\n string filename = 5;\n}\n\nmessage Info {\n uint32 handle = 1;\n uint64 sysv_key = 2;\n repeated Image images = 3;\n repeated string labels = 4;\n}\n\nmessage InfoRequest {\n uint32 image_size = 1;\n}\n" }, { "path": "examples/11_Protos/demo/inference.proto", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n\nsyntax = \"proto3\";\n\npackage ssd;\n\nservice Inference {\n rpc Compute (BatchInput) returns (BatchPredictions) {}\n}\n\nmessage BatchInput {\n uint32 engine_id = 1;\n uint64 batch_id = 2;\n uint32 batch_size = 3;\n\n uint32 int_offset = 4;\n uint64 sysv_offset = 5;\n bytes data = 6;\n}\n\nmessage BatchPredictions {\n repeated Element elements = 1;\n uint64 batch_id = 2;\n float compute_time = 3;\n float total_time = 4;\n repeated Timer timers = 5;\n}\n\nmessage Element {\n repeated Prediction predictions = 2;\n}\n\nmessage Bbox {\n float x = 1; // upper left\n float y = 2; // upper left\n float width = 3;\n float height = 4;\n uint32 class_id = 5;\n}\n\nmessage Prediction {\n uint32 class_id = 1;\n string class_str = 2;\n float score = 3;\n Bbox bbox = 4;\n}\n\nmessage Timer {\n string name = 1;\n float time = 2;\n enum TimerUnit {\n SECONDS = 0;\n MILLI = 1;\n MICRO = 2;\n NANO = 3;\n }\n TimerUnit unit = 3;\n}\n" }, { "path": "examples/11_Protos/echo/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nINCLUDE(GRPCGenerateCPP)\n\nPROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS \n echo.proto\n)\n\nPROTOBUF_GENERATE_GRPC_CPP(PROTO_GRPC_SRCS PROTO_GRPC_HDRS \n echo.proto\n)\n\nadd_library(echo-protos\n ${PROTO_SRCS}\n ${PROTO_GRPC_SRCS}\n)\n\ntarget_link_libraries(echo-protos\n PUBLIC\n ${_PROTOBUF_LIBPROTOBUF}\n)\n\ntarget_include_directories(echo-protos PUBLIC\n ${CMAKE_CURRENT_BINARY_DIR}\n)\n\n" }, { "path": "examples/11_Protos/echo/echo.proto", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n\nsyntax = \"proto3\";\n\npackage simple;\n\nservice Inference {\n rpc Compute (Input) returns (Output) {}\n rpc Bidirectional (stream Input) returns (stream Output) {}\n rpc BatchedCompute (stream Input) returns (stream Output) {}\n}\n\nmessage SystemV {\n uint64 shm_id = 1;\n uint64 offset = 2;\n uint64 size = 3;\n}\n\nmessage Input {\n uint64 batch_id = 1;\n oneof data {\n bytes raw_bytes = 2;\n SystemV sysv = 3;\n }\n}\n\nmessage Output {\n uint64 batch_id = 1;\n}\n\n" }, { "path": "examples/11_Protos/inference/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nINCLUDE(GRPCGenerateCPP)\n\nPROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS \n nvidia_inference.proto\n api.proto\n model_config.proto\n request_status.proto\n server_status.proto\n)\n\nPROTOBUF_GENERATE_GRPC_CPP(PROTO_GRPC_SRCS PROTO_GRPC_HDRS \n nvidia_inference.proto\n)\n\nadd_library(nv-inference-protos\n ${PROTO_SRCS}\n ${PROTO_GRPC_SRCS}\n)\n\ntarget_link_libraries(nv-inference-protos\n PUBLIC\n ${_PROTOBUF_LIBPROTOBUF}\n)\n\ntarget_include_directories(nv-inference-protos PUBLIC\n ${CMAKE_CURRENT_BINARY_DIR}\n)\n\n" }, { "path": "examples/11_Protos/inference/api.proto", "content": "// Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n//\n// Redistribution and use in source and binary forms, with or without\n// modification, are permitted provided that the following conditions\n// are met:\n// * Redistributions of source code must retain the above copyright\n// notice, this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright\n// notice, this list of conditions and the following disclaimer in the\n// documentation and/or other materials provided with the distribution.\n// * Neither the name of NVIDIA CORPORATION nor the names of its\n// contributors may be used to endorse or promote products derived\n// from this software without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nsyntax = \"proto3\";\n\npackage nvidia.inferenceserver;\n\n// Request header for inferencing. The actual input data is delivered\n// separate from the header.\nmessage InferRequestHeader {\n // Input...\n message Input {\n // Name of the input.\n string name = 1;\n\n // Size of the input, in bytes. This is the size for one instance\n // of the input, not the entire size of a batch of the input.\n uint64 byte_size = 2;\n }\n\n // Output...\n message Output {\n // Name of the output.\n string name = 1;\n\n // Size of the output, in bytes. This is the size for one instance\n // of the output, not the entire size of a batch of the output.\n uint64 byte_size = 2;\n\n // Class result format. The output must be a vector. Output values\n // will be interpreted as probabilities and the highest 'count'\n // values will be returned.\n message Class {\n // Return the 'count' highest valued results.\n uint32 count = 1;\n }\n\n // Optional. If defined return this result as a classification\n // instead of raw data.\n Class cls = 3;\n }\n\n // Batch size of the inference inputs.\n uint32 batch_size = 1;\n\n // Inference inputs.\n repeated Input input = 2;\n\n // Inference outputs that are being requested.\n repeated Output output = 3;\n}\n\n// Response header for inferencing. Any raw response data (i.e. tensor\n// values) is delivered separately from the header.\nmessage InferResponseHeader {\n // Output...\n message Output {\n // Name of the output.\n string name = 1;\n\n // Raw result\n message Raw {\n // Size of the output, in bytes. This is the size for one\n // instance of the output, not the entire size of a batch of the\n // output.\n uint64 byte_size = 1;\n }\n\n // Classification result\n message Class {\n // The index in the output tensor.\n int32 idx = 1;\n // The value of the class as a float (typically a probability).\n float value = 2;\n // The label for the class (optional, only available if provided\n // by the model).\n string label = 3;\n }\n message Classes {\n // The topk classes for this output\n repeated Class cls = 1;\n }\n\n // Result format for this output. Only one of these may be\n // specified. For 'batch_classes' there should be one entry for\n // each output of the batch.\n Raw raw = 2;\n repeated Classes batch_classes = 3;\n }\n\n // Name of the model that produced the results.\n string model_name = 1;\n\n // Version of the model that produced the results.\n uint32 model_version = 2;\n\n // Batch size of the inference outputs.\n uint32 batch_size = 3;\n\n // The outputs\n repeated Output output = 4;\n}\n" }, { "path": "examples/11_Protos/inference/model_config.proto", "content": "// Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n//\n// Redistribution and use in source and binary forms, with or without\n// modification, are permitted provided that the following conditions\n// are met:\n// * Redistributions of source code must retain the above copyright\n// notice, this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright\n// notice, this list of conditions and the following disclaimer in the\n// documentation and/or other materials provided with the distribution.\n// * Neither the name of NVIDIA CORPORATION nor the names of its\n// contributors may be used to endorse or promote products derived\n// from this software without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n//\n// Copyright (c) 2018-2019, TensorFlow Authors. All rights reserved.\n\nsyntax = \"proto3\";\n\npackage nvidia.inferenceserver;\n\n// Data types supported for input and output tensors.\nenum DataType {\n TYPE_INVALID = 0;\n\n TYPE_BOOL = 1;\n\n TYPE_UINT8 = 2;\n TYPE_UINT16 = 3;\n TYPE_UINT32 = 4;\n TYPE_UINT64 = 5;\n\n TYPE_INT8 = 6;\n TYPE_INT16 = 7;\n TYPE_INT32 = 8;\n TYPE_INT64 = 9;\n\n TYPE_FP16 = 10;\n TYPE_FP32 = 11;\n TYPE_FP64 = 12;\n}\n\n// A group of one or more instances of a model and resources made\n// available for those instances.\nmessage ModelInstanceGroup {\n // Kind of this instance group.\n enum Kind {\n // This instance group represents instances that can run on either\n // CPU or GPU. If all GPUs listed in 'gpus' are available then\n // instances will be create on GPU(s), otherwise instances will be\n // created on CPU.\n KIND_AUTO = 0;\n\n // This instance group represents instances that must run on the\n // GPU.\n KIND_GPU = 1;\n\n // This instance group represents instances that must run on the\n // CPU.\n KIND_CPU = 2;\n }\n\n // Optional name of this group of instances. If not specified the\n // name will be formed as _. The name of\n // individual instances will be further formed by a unique instance\n // number and GPU index:\n // __gpu\n string name = 1;\n\n // The kind of this instance group. Default is KIND_AUTO. If\n // KIND_AUTO or KIND_GPU then both 'count' and 'gpu' are valid and\n // may be specified. If KIND_CPU only 'count' is valid and 'gpu'\n // cannot be specified.\n Kind kind = 4;\n\n // Number of instances in this group created for each GPU listed in\n // 'gpus'. Default is 1.\n int32 count = 2;\n\n // GPU(s) where instances should be available. For each GPU listed,\n // 'count' instances of the model will be available. Setting 'gpus'\n // to empty (or not specifying at all) is eqivalent to listing all\n // system GPUs.\n repeated int32 gpus = 3;\n}\n\n// Input tensor for the model\nmessage ModelInput {\n // Format for the input.\n enum Format {\n // The input has no specific format.\n FORMAT_NONE = 0;\n\n // Image formats. Tensors with this format require 3 dimensions if\n // the model does not support batching (max_batch_size = 0) or 4\n // dimensions if the model does support batching (max_batch_size\n // >= 1). In either case the 'dims' below should only specify the\n // 3 non-batch dimensions (i.e. HWC or CHW).\n FORMAT_NHWC = 1;\n FORMAT_NCHW = 2;\n }\n\n string name = 1;\n DataType data_type = 2;\n Format format = 3;\n repeated int64 dims = 4;\n}\n\n// Output tensor for the model\nmessage ModelOutput {\n string name = 1;\n DataType data_type = 2;\n repeated int64 dims = 3;\n\n // Label file for this output (optional).\n string label_filename = 4;\n}\n\n// Policy indicating which versions of a model should be made\n// available by the inference server.\nmessage ModelVersionPolicy {\n // Serve only the 'num_versions' highest-numbered versions. This is\n // the default policy and the default value of 'num_versions' is 1,\n // indicating that by default only the highest-number version of a\n // model will be served.\n message Latest {\n uint32 num_versions = 1;\n }\n\n // Serve all versions of the model.\n message All {\n }\n\n // Serve only a specific set of versions of the model.\n message Specific {\n repeated int64 versions = 1;\n }\n\n // Each model must implement only a single policy. The default\n // policy is 'Latest'.\n oneof policy_choice {\n Latest latest = 1;\n All all = 2;\n Specific specific = 3;\n }\n}\n\n// Model configuration.\nmessage ModelConfig {\n // Name of the model.\n string name = 1;\n\n // Type of model (e.g. \"tensorflow\").\n string platform = 2;\n\n // Policy indicating which version(s) of the model will be served.\n ModelVersionPolicy version_policy = 3;\n\n // Maximum batch size allowed for inference. This can only decrease\n // what is allowed by the model itself. A value of 0 indicates that\n // batching is not-allowed/is-disabled (for some input formats this\n // has implications on the expected dimension of the inputs, see\n // Format above).\n int32 max_batch_size = 4;\n\n // Inputs and outputs to the model.\n repeated ModelInput input = 5;\n repeated ModelOutput output = 6;\n\n // Optional instances of this model. If not specified, one instance\n // of the model will be instantiated on each available GPU.\n repeated ModelInstanceGroup instance_group = 7;\n\n // Optional filename of the model file to use if a\n // compute-capability specific model is not specified in\n // 'cc_model_names'. If not specified the default is model.graphdef\n // for TF graphdef models and model.plan for TensorRT PLAN models.\n string default_model_filename = 8;\n\n // Optional map from CUDA compute capabilities to the filename of\n // the model that supports that compute capability. The filename\n // refers to a file within the model version directory.\n map cc_model_filenames = 9;\n }\n\n// List of model configurations.\nmessage ModelConfigList {\n repeated ModelConfig config = 1;\n}\n" }, { "path": "examples/11_Protos/inference/nvidia_inference.proto", "content": "// Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n//\n// Redistribution and use in source and binary forms, with or without\n// modification, are permitted provided that the following conditions\n// are met:\n// * Redistributions of source code must retain the above copyright\n// notice, this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright\n// notice, this list of conditions and the following disclaimer in the\n// documentation and/or other materials provided with the distribution.\n// * Neither the name of NVIDIA CORPORATION nor the names of its\n// contributors may be used to endorse or promote products derived\n// from this software without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nsyntax = \"proto3\";\n\npackage nvidia.inferenceserver;\n\nimport \"api.proto\";\nimport \"request_status.proto\";\nimport \"server_status.proto\";\n\nservice GRPCService {\n // Get server or model status\n rpc Status(StatusRequest) returns (StatusResponse) {}\n\n // Control server profiling\n rpc Profile(ProfileRequest) returns (ProfileResponse) {}\n\n // Health check\n rpc Health(HealthRequest) returns (HealthResponse) {}\n\n // Perform inference. [ Set the maximum message size (default 4 MB)\n // and transmit in one pass Seems like Tensorflow uses this approach\n // to transfer tensor which can be large\n // https://github.com/grpc/grpc/issues/8975 ]\n rpc Infer(InferRequest) returns (InferResponse) {}\n}\n\n// Request message for server status.\nmessage StatusRequest {\n // The specific model status to be returned. Return status for all\n // models if empty.\n string model_name = 1;\n}\n\n// Response message for server status.\nmessage StatusResponse {\n RequestStatus request_status = 1;\n ServerStatus server_status = 2;\n}\n\n// Request message for profile.\nmessage ProfileRequest {\n string cmd = 1;\n}\n\n// Response message for profile.\nmessage ProfileResponse{\n RequestStatus request_status = 1;\n}\n\n// Request message for health.\nmessage HealthRequest {\n string mode = 1;\n}\n\n// Response message for health.\nmessage HealthResponse{\n RequestStatus request_status = 1;\n bool health = 2;\n}\n\n// Request message for inference.\nmessage InferRequest {\n // Name of model to use for inference\n string model_name = 1;\n\n // Version of the model to use for inference. If not specified use\n // the latest/most-recent version of the model. [ Use string here so\n // default value of empty indicates not specified].\n string version = 2;\n\n // Meta-data for the inference request.\n InferRequestHeader meta_data = 3;\n\n // Raw input tensor data in the order specified in 'meta_data'.\n repeated bytes raw_input = 4;\n\n uint64 batch_id = 100;\n uint32 batch_size = 101;\n uint64 sysv_offset = 102;\n}\n\n// Response message for inference.\nmessage InferResponse {\n RequestStatus request_status = 1;\n\n // Meta-data for the inference response.\n InferResponseHeader meta_data = 2;\n\n // Raw output tensor data in the order specified in 'meta_data'.\n repeated bytes raw_output = 3;\n\n uint64 batch_id = 100;\n float compute_time = 101;\n float request_time = 102;\n}\n" }, { "path": "examples/11_Protos/inference/request_status.proto", "content": "// Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n//\n// Redistribution and use in source and binary forms, with or without\n// modification, are permitted provided that the following conditions\n// are met:\n// * Redistributions of source code must retain the above copyright\n// notice, this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright\n// notice, this list of conditions and the following disclaimer in the\n// documentation and/or other materials provided with the distribution.\n// * Neither the name of NVIDIA CORPORATION nor the names of its\n// contributors may be used to endorse or promote products derived\n// from this software without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nsyntax = \"proto3\";\n\npackage nvidia.inferenceserver;\n\n// Status codes returned for inference server requests\nenum RequestStatusCode {\n INVALID = 0;\n\n SUCCESS = 1;\n UNKNOWN = 2;\n INTERNAL = 3;\n NOT_FOUND = 4;\n INVALID_ARG = 5;\n UNAVAILABLE = 6;\n UNSUPPORTED = 7;\n}\n\n// Status returned for all inference server requests\nmessage RequestStatus {\n // Required status code\n RequestStatusCode code = 1;\n\n // Optional message\n string msg = 2;\n\n // Inference server identifier.\n string server_id = 3;\n\n // Unique identifier for the request. Value 0 (zero) indicates\n // request ID is not known.\n uint64 request_id = 4;\n}\n" }, { "path": "examples/11_Protos/inference/server_status.proto", "content": "// Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n//\n// Redistribution and use in source and binary forms, with or without\n// modification, are permitted provided that the following conditions\n// are met:\n// * Redistributions of source code must retain the above copyright\n// notice, this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above copyright\n// notice, this list of conditions and the following disclaimer in the\n// documentation and/or other materials provided with the distribution.\n// * Neither the name of NVIDIA CORPORATION nor the names of its\n// contributors may be used to endorse or promote products derived\n// from this software without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nsyntax = \"proto3\";\n\npackage nvidia.inferenceserver;\n\nimport \"model_config.proto\";\n\n// Statistic collecting a duration metric\nmessage StatDuration {\n // Cummulative number of times this metric occurred.\n uint64 count = 1;\n\n // Total collected duration of this metric in nanoseconds.\n uint64 total_time_ns = 2;\n}\n\n// Stats collected for Status requests.\nmessage StatusRequestStats {\n // Total time required to service api/status requests, not including\n // HTTP or gRPC endpoint termination time.\n StatDuration success = 1;\n}\n\n// Stats collected for Profile requests.\nmessage ProfileRequestStats {\n // Total time required to service profile requests, not including\n // HTTP or gRPC endpoint termination time.\n StatDuration success = 1;\n}\n\n// Stats collected for Health requests.\nmessage HealthRequestStats {\n // Total time required to service health requests, not including\n // HTTP or gRPC endpoint termination time.\n StatDuration success = 1;\n}\n\n// Stats collected for Infer requests.\nmessage InferRequestStats {\n // Total time required to service successful inference requests,\n // not including HTTP or gRPC endpoint termination time.\n StatDuration success = 1;\n\n // Total time required to service failed inference requests, not\n // including HTTP or gRPC endpoint termination time.\n StatDuration failed = 2;\n\n // Time required to run inferencing including time waiting for an\n // available model instance, time copying input tensors to GPU\n // memory, time executing the model, and time copying output tensors\n // from GPU memory. Wait time is also captured separately 'run_wait'\n // so to get inferencing time not including wait time use 'run' -\n // 'run_wait'.\n StatDuration run = 3;\n\n // Time waiting for an available model instance.\n StatDuration run_wait = 4;\n}\n\n// Server readiness states.\nenum ModelReadyState {\n MODEL_UNKNOWN = 0;\n MODEL_READY = 1;\n MODEL_UNAVAILABLE = 2;\n MODEL_LOADING = 3;\n MODEL_UNLOADING = 4;\n}\n\n// Status for a version of a model.\nmessage ModelVersionStatus {\n // Current readiness state for the model version.\n ModelReadyState ready_state = 1;\n\n // Duration statistics for each batch size used for this version of\n // the model.\n map infer_stats = 2;\n}\n\n// Status for a model.\nmessage ModelStatus {\n // The configuration for the model.\n ModelConfig config = 1;\n\n // Duration statistics for each version of this model.\n map version_status = 2;\n}\n\n// Server readiness states.\nenum ServerReadyState {\n SERVER_INVALID = 0;\n SERVER_INITIALIZING = 1;\n SERVER_READY = 2;\n SERVER_EXITING = 3;\n\n SERVER_FAILED_TO_INITIALIZE = 10;\n}\n\n// Status for inference server\nmessage ServerStatus {\n // Server ID.\n string id = 1;\n\n // Server version.\n string version = 2;\n\n // Current readiness state for the server.\n ServerReadyState ready_state = 7;\n\n // Server uptime in nanoseconds\n uint64 uptime_ns = 3;\n\n // Status for each model on the server as map from ->\n // ModelStatus.\n map model_status = 4;\n\n // Statistics for Status requests.\n StatusRequestStats status_stats = 5;\n\n // Statistics for Profile requests.\n ProfileRequestStats profile_stats = 6;\n\n // Statistics for Health requests.\n HealthRequestStats health_stats = 8;\n}\n" }, { "path": "examples/12_ConfigGenerator/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\npybind11_add_module(config_generator generator.cc)\n\ntarget_link_libraries(config_generator \n PUBLIC\n yais\n nv-inference-protos\n)\n" }, { "path": "examples/12_ConfigGenerator/README.md", "content": "# TensorRT Inference Server Model Store Builder\n\n- Ensure you built the project.\n- Run `./link.sh` in this directory\n\n## Design Requirements\n\nThis example consists of a ModelStore manager (Python) and a\nModelConfigGenerator (C++ w/ Python bindings).\n\nThe ModelConfigGenerator shall:\n - [X] parse serialized TensorRT engine files\n - [X] translate the necessary properties of the ICudaEngine to an\n `::nvidia::inferenceserver::ModelConfig` protobuf message\n - [ ] not require the presence of Cuda or a GPU to perform the actions\n\nThe ModelStore manager consists of a Python class for direct consumption and a\ncommand-line application that shall:\n - [ ] create and manage a model-store in a user-supplied filesystem directory \n - [X] add TensorRT model files to the model store using the\n ModelConfigGenerator and user-specified arguments\n - [ ] add new version of TensorRT models to a ModelStore\n - [ ] remove versions of entire models from the ModelStore\n - [ ] add, edit, update and remove Tensorflow models\n - [ ] add, edit, update and remove PyTorch/Caffe2 models\n\n## Prototype Implementation\n\n```\n./ms_mgmt --help\nUsage: ms_mgmt [OPTIONS]\n\nOptions:\n --engine PATH TensorRT serialized engine [required]\n --concurrency INTEGER max number of concurrency executions allowed\n --name TEXT model name; default to basename(engine) with the ext\n dropped\n --version INTEGER model version\n --store-path TEXT model store path; default to ./model-store\n --help Show this message and exit.\n```\n\n```\n./ms_mgmt --store-path=/tmp/model-store --engine=/work/models/ResNet-50-b1-fp32.engine --name=overridden-model-name --version=1337 --concurrency=10\n\nls /tmp/model-store/\noverridden-model-name\n\nls /tmp/model-store/overridden-model-name/1337/\nResNet-50-b1-fp32.engine model.plan\n\ncat /tmp/model-store/overridden-model-name/config.pbtxt\nname: \"overridden-model-name\"\nplatform: \"tensorrt_plan\"\nmax_batch_size: 1\ninput {\n name: \"data\"\n data_type: TYPE_FP32\n dims: 3\n dims: 224\n dims: 224\n}\noutput {\n name: \"prob\"\n data_type: TYPE_FP32\n dims: 1000\n dims: 1\n dims: 1\n}\ninstance_group {\n count: 10\n gpus: 0\n}\n```\n" }, { "path": "examples/12_ConfigGenerator/generator.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n\n#include \n#include \n\n#include \"YAIS/TensorRT/TensorRT.h\"\n#include \"YAIS/YAIS.h\"\n\n// NVIDIA Inference Server Protos\n#include \"nvidia_inference.grpc.pb.h\"\n#include \"nvidia_inference.pb.h\"\n\nusing nvidia::inferenceserver::ModelConfig;\n\nusing trtlab::TensorRT::Runtime;\n\nstatic size_t DataTypeToBytes(nvidia::inferenceserver::DataType dataType)\n{\n switch(dataType)\n {\n case nvidia::inferenceserver::TYPE_INVALID:\n CHECK(false) << \"Invalid DataType used\";\n return 0;\n case nvidia::inferenceserver::TYPE_BOOL:\n case nvidia::inferenceserver::TYPE_UINT8:\n case nvidia::inferenceserver::TYPE_INT8:\n return 1;\n case nvidia::inferenceserver::TYPE_UINT16:\n case nvidia::inferenceserver::TYPE_INT16:\n case nvidia::inferenceserver::TYPE_FP16:\n return 2;\n case nvidia::inferenceserver::TYPE_UINT32:\n case nvidia::inferenceserver::TYPE_INT32:\n case nvidia::inferenceserver::TYPE_FP32:\n return 4;\n case nvidia::inferenceserver::TYPE_UINT64:\n case nvidia::inferenceserver::TYPE_INT64:\n case nvidia::inferenceserver::TYPE_FP64:\n return 8;\n default:\n CHECK(false) << \"Invalid DataType used\";\n return 0;\n }\n}\n\nstatic nvidia::inferenceserver::DataType ConvertTensorRTDataType(nvinfer1::DataType trt_datatype)\n{\n switch(trt_datatype)\n {\n case nvinfer1::DataType::kFLOAT:\n return nvidia::inferenceserver::TYPE_FP32;\n case nvinfer1::DataType::kHALF:\n return nvidia::inferenceserver::TYPE_FP16;\n case nvinfer1::DataType::kINT8:\n return nvidia::inferenceserver::TYPE_INT8;\n case nvinfer1::DataType::kINT32:\n return nvidia::inferenceserver::TYPE_INT32;\n default:\n LOG(FATAL) << \"Unknown TensorRT DataType\";\n }\n}\n\nstd::string tensorrt_engine(std::string model_name, std::string engine, int concurrency)\n{\n ModelConfig config;\n auto model = trtlab::TensorRT::Runtime::DeserializeEngine(engine);\n config.set_name(model_name);\n config.set_platform(\"tensorrt_plan\");\n config.set_max_batch_size(model->GetMaxBatchSize());\n\n for(auto i : model->GetInputBindingIds())\n {\n const auto& binding = model->GetBinding(i);\n auto input = config.add_input();\n input->set_name(binding.name);\n input->set_data_type(ConvertTensorRTDataType(binding.dtype));\n for(auto d : binding.dims)\n {\n input->add_dims(d);\n }\n }\n\n for(auto i : model->GetOutputBindingIds())\n {\n const auto& binding = model->GetBinding(i);\n auto output = config.add_output();\n output->set_name(binding.name);\n output->set_data_type(ConvertTensorRTDataType(binding.dtype));\n for(auto d : binding.dims)\n {\n output->add_dims(d);\n }\n }\n\n auto instance_group = config.add_instance_group();\n CHECK(concurrency > 0) << \"Concurrency must be >= 0\";\n instance_group->set_count(concurrency);\n instance_group->add_gpus(0);\n\n return config.DebugString();\n}\n\nnamespace py = pybind11;\n\nPYBIND11_MODULE(config_generator, m)\n{\n m.doc() = R\"pbdoc(\n Pybind11 Yais plugin\n --------------------\n .. currentmodule:: config_generator\n .. autosummary::\n :toctree: _generate\n tensorrt_engine\n )pbdoc\";\n\n m.def(\"tensorrt_engine\", &tensorrt_engine, R\"pbdoc(\n Generate a TensorRT Inference Server ModelConfig from a serialized engine file\n )pbdoc\");\n\n#ifdef VERSION_INFO\n m.attr(\"__version__\") = VERSION_INFO;\n#else\n m.attr(\"__version__\") = \"dev\";\n#endif\n}\n" }, { "path": "examples/12_ConfigGenerator/link.sh", "content": "#!/bin/bash\n\nln -s /work/build/examples/12_ConfigGenerator/config_generator.cpython-35m-x86_64-linux-gnu.so\n" }, { "path": "examples/12_ConfigGenerator/ms_mgmt", "content": "#!/usr/bin/env python3\nimport os\nimport pathlib\nimport shutil\n\nfrom contextlib import contextmanager\n\nimport click\nimport config_generator as cg\n\nFileType = click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True)\nPathType = click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)\n\n@contextmanager\ndef cd(newdir):\n prevdir = os.getcwd()\n os.chdir(os.path.expanduser(newdir))\n try:\n yield\n finally:\n os.chdir(prevdir)\n\n# Path(exists=False, file_okay=True, dir_okay=True, writable=False, readable=True, resolve_path=False)\n\nclass ModelStore:\n\n def __init__(self, *, path, create=False, validate=False):\n if not os.path.isdir(path):\n pathlib.Path(path).mkdir(parents=create, exist_ok=create)\n self.path = os.path.abspath(path)\n self.name = os.path.basename(self.path)\n\n def model_name(self, *, engine, name=None):\n base = os.path.basename(engine)\n model = name or base.replace(\".engine\", \"\")\n return model\n\n def model_path(self, *, name):\n return os.path.join(self.path, name)\n\n def engine_path(self, *, name, version=0):\n return os.path.join(self.model_path(name=name), str(version))\n\n def create_engine_path(self, *, name, version):\n engine_path = self.engine_path(name=name, version=version)\n if os.path.exists(engine_path):\n raise RuntimeError(\"{} already exists in the model store\".format(model))\n pathlib.Path(engine_path).mkdir(parents=True, exist_ok=True)\n return engine_path\n\n def copy_and_link_engine(self, *, name, version, engine):\n engine_path = self.create_engine_path(name=name, version=version)\n shutil.copy(engine, engine_path)\n with cd(engine_path):\n os.symlink(os.path.basename(engine), \"model.plan\")\n\n def add_tensorrt_engine(self, *, engine, name=None, concurrency=1, version=0):\n engine = os.path.abspath(engine)\n if not os.path.isfile(engine):\n raise RuntimeError(\"{} engine does not exist\".format(engine))\n name = self.model_name(engine=engine, name=name)\n model_path = self.model_path(name=name)\n self.copy_and_link_engine(name=name, version=version, engine=engine)\n config = cg.tensorrt_engine(name, engine, concurrency)\n with cd(model_path), open(\"config.pbtxt\", \"w\") as file:\n file.write(config)\n\n@click.command()\n@click.option(\"--engine\", type=FileType, required=True, help=\"TensorRT serialized engine\")\n@click.option(\"--concurrency\", type=int, default=1, help=\"max number of concurrency executions allowed\")\n@click.option(\"--name\", default=None, help=\"model name; default to basename(engine) with the ext dropped\")\n@click.option(\"--version\", type=int, default=0, help=\"model version\")\n@click.option(\"--store-path\", default=None, help=\"model store path; default to ./model-store\")\ndef main(engine, concurrency, name, store_path, version):\n store_path = store_path or \"model-store\"\n store = ModelStore(path=store_path, create=True)\n base = os.path.basename(engine)\n name = name or base.replace(\".engine\", \"\")\n store.add_tensorrt_engine(engine=engine, name=name, concurrency=concurrency, version=version)\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "examples/12_FlatBuffers/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nfind_package(Flatbuffers)\n\n# This module defines\n# FLATBUFFERS_INCLUDE_DIR, directory containing headers\n# FLATBUFFERS_LIBS, directory containing flatbuffers libraries\n# FLATBUFFERS_STATIC_LIB, path to libflatbuffers.a\n# FLATBUFFERS_FOUND, whether flatbuffers has been found\n\n\nadd_library(example-fbs \n example.grpc.fb.cc\n)\n\ntarget_link_libraries(example-fbs PUBLIC\n flatbuffers\n)\n\ntarget_include_directories(example-fbs PUBLIC\n ${CMAKE_CURRENT_BINARY_DIR}\n)\n\n\nadd_executable(fb-server.x\n server.cc)\n\ntarget_link_libraries(fb-server.x\n nvrpc\n example-fbs\n gflags\n)\n\nadd_executable(fb-client.x\n client.cc)\n\ntarget_link_libraries(fb-client.x\n nvrpc\n example-fbs\n gflags\n)\n" }, { "path": "examples/12_FlatBuffers/client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"example.grpc.fb.h\"\n#include \"example_generated.h\"\n\nusing Input = flatbuffers::grpc::Message;\nusing Output = flatbuffers::grpc::Message;\n\nusing grpc::Channel;\nusing grpc::ClientContext;\nusing grpc::Status;\n\nclass SimpleClient\n{\n public:\n SimpleClient(std::shared_ptr channel) : stub_(Greeter::NewStub(channel)) {}\n\n // Assembles the client's payload, sends it and presents the response back\n // from the server.\n std::string Compute(const int batch_id)\n {\n flatbuffers::grpc::MessageBuilder mb;\n\n // Data we are sending to the server.\n auto name_offset = mb.CreateString(std::to_string(batch_id));\n auto request_offset = CreateHelloRequest(mb, name_offset);\n mb.Finish(request_offset);\n auto request = mb.ReleaseMessage();\n\n // Container for the data we expect from the server.\n Output reply;\n\n // Context for the client. It could be used to convey extra information to\n // the server and/or tweak certain RPC behaviors.\n ClientContext context;\n\n // The actual RPC.\n Status status = stub_->SayHello(&context, request, &reply);\n\n // Act upon its status.\n if(status.ok())\n {\n const HelloReply* output = reply.GetRoot();\n return output->message()->str();\n }\n else\n {\n std::cout << status.error_code() << \": \" << status.error_message() << std::endl;\n return \"Fail!\";\n }\n }\n\n private:\n std::unique_ptr stub_;\n};\n\nDEFINE_int32(count, 100, \"number of grpc messages to send\");\n\nint main(int argc, char** argv)\n{\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n SimpleClient client(grpc::CreateChannel(\"localhost:50051\", grpc::InsecureChannelCredentials()));\n auto start = std::chrono::steady_clock::now();\n for(int i = 0; i < FLAGS_count; i++)\n {\n auto reply = client.Compute(i);\n LOG_FIRST_N(INFO, 20) << reply;\n }\n auto end = std::chrono::steady_clock::now();\n float elapsed = std::chrono::duration(end - start).count();\n std::cout << FLAGS_count << \" requests in \" << elapsed << \"seconds\" << std::endl;\n return 0;\n}\n" }, { "path": "examples/12_FlatBuffers/example.fbs", "content": "table HelloReply {\n message:string;\n}\n\ntable HelloRequest {\n name:string;\n}\n\ntable ManyHellosRequest {\n name:string;\n num_greetings:int;\n}\n\nrpc_service Greeter {\n SayHello(HelloRequest):HelloReply;\n SayManyHellos(ManyHellosRequest):HelloReply (streaming: \"server\");\n}\n" }, { "path": "examples/12_FlatBuffers/example.grpc.fb.cc", "content": "// Generated by the gRPC C++ plugin.\n// If you make any local change, they will be lost.\n// source: example\n\n#include \"example.grpc.fb.h\"\n#include \"example_generated.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nstatic const char* Greeter_method_names[] = {\n \"/Greeter/SayHello\",\n \"/Greeter/SayManyHellos\",\n};\n\nstd::unique_ptr\n Greeter::NewStub(const std::shared_ptr<::grpc::ChannelInterface>& channel,\n const ::grpc::StubOptions& options)\n{\n std::unique_ptr stub(new Greeter::Stub(channel));\n return stub;\n}\n\nGreeter::Stub::Stub(const std::shared_ptr<::grpc::ChannelInterface>& channel)\n : channel_(channel), rpcmethod_SayHello_(Greeter_method_names[0],\n ::grpc::internal::RpcMethod::NORMAL_RPC, channel),\n rpcmethod_SayManyHellos_(Greeter_method_names[1],\n ::grpc::internal::RpcMethod::SERVER_STREAMING, channel)\n{\n}\n\n::grpc::Status Greeter::Stub::SayHello(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n flatbuffers::grpc::Message* response)\n{\n return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SayHello_, context,\n request, response);\n}\n\n::grpc::ClientAsyncResponseReader>*\n Greeter::Stub::AsyncSayHelloRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq)\n{\n return ::grpc::internal::ClientAsyncResponseReaderFactory<\n flatbuffers::grpc::Message>::Create(channel_.get(), cq, rpcmethod_SayHello_,\n context, request, true);\n}\n\n::grpc::ClientAsyncResponseReader>*\n Greeter::Stub::PrepareAsyncSayHelloRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq)\n{\n return ::grpc::internal::ClientAsyncResponseReaderFactory<\n flatbuffers::grpc::Message>::Create(channel_.get(), cq, rpcmethod_SayHello_,\n context, request, false);\n}\n\n::grpc::ClientReader>*\n Greeter::Stub::SayManyHellosRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request)\n{\n return ::grpc::internal::ClientReaderFactory>::Create(\n channel_.get(), rpcmethod_SayManyHellos_, context, request);\n}\n\n::grpc::ClientAsyncReader>*\n Greeter::Stub::AsyncSayManyHellosRaw(\n ::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request, ::grpc::CompletionQueue* cq,\n void* tag)\n{\n return ::grpc::internal::ClientAsyncReaderFactory<\n flatbuffers::grpc::Message>::Create(channel_.get(), cq,\n rpcmethod_SayManyHellos_, context, request,\n true, tag);\n}\n\n::grpc::ClientAsyncReader>*\n Greeter::Stub::PrepareAsyncSayManyHellosRaw(\n ::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request, ::grpc::CompletionQueue* cq)\n{\n return ::grpc::internal::ClientAsyncReaderFactory<\n flatbuffers::grpc::Message>::Create(channel_.get(), cq,\n rpcmethod_SayManyHellos_, context, request,\n false, nullptr);\n}\n\nGreeter::Service::Service()\n{\n AddMethod(new ::grpc::internal::RpcServiceMethod(\n Greeter_method_names[0], ::grpc::internal::RpcMethod::NORMAL_RPC,\n new ::grpc::internal::RpcMethodHandler,\n flatbuffers::grpc::Message>(\n std::mem_fn(&Greeter::Service::SayHello), this)));\n AddMethod(new ::grpc::internal::RpcServiceMethod(\n Greeter_method_names[1], ::grpc::internal::RpcMethod::SERVER_STREAMING,\n new ::grpc::internal::ServerStreamingHandler,\n flatbuffers::grpc::Message>(\n std::mem_fn(&Greeter::Service::SayManyHellos), this)));\n}\n\nGreeter::Service::~Service() {}\n\n::grpc::Status Greeter::Service::SayHello(::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n flatbuffers::grpc::Message* response)\n{\n (void)context;\n (void)request;\n (void)response;\n return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n}\n\n::grpc::Status Greeter::Service::SayManyHellos(\n ::grpc::ServerContext* context, const flatbuffers::grpc::Message* request,\n ::grpc::ServerWriter>* writer)\n{\n (void)context;\n (void)request;\n (void)writer;\n return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n}\n" }, { "path": "examples/12_FlatBuffers/example.grpc.fb.h", "content": "// Generated by the gRPC C++ plugin.\n// If you make any local change, they will be lost.\n// source: example\n#ifndef GRPC_example__INCLUDED\n#define GRPC_example__INCLUDED\n\n#include \"example_generated.h\"\n#include \"flatbuffers/grpc.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace grpc {\nclass CompletionQueue;\nclass Channel;\nclass ServerCompletionQueue;\nclass ServerContext;\n} // namespace grpc\n\nclass Greeter final\n{\n public:\n static constexpr char const* service_full_name() { return \"Greeter\"; }\n class StubInterface\n {\n public:\n virtual ~StubInterface() {}\n virtual ::grpc::Status SayHello(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n flatbuffers::grpc::Message* response) = 0;\n std::unique_ptr<\n ::grpc::ClientAsyncResponseReaderInterface>>\n AsyncSayHello(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq)\n {\n return std::unique_ptr<\n ::grpc::ClientAsyncResponseReaderInterface>>(\n AsyncSayHelloRaw(context, request, cq));\n }\n std::unique_ptr<\n ::grpc::ClientAsyncResponseReaderInterface>>\n PrepareAsyncSayHello(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq)\n {\n return std::unique_ptr<\n ::grpc::ClientAsyncResponseReaderInterface>>(\n PrepareAsyncSayHelloRaw(context, request, cq));\n }\n std::unique_ptr<::grpc::ClientReaderInterface>>\n SayManyHellos(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request)\n {\n return std::unique_ptr<\n ::grpc::ClientReaderInterface>>(\n SayManyHellosRaw(context, request));\n }\n std::unique_ptr<::grpc::ClientAsyncReaderInterface>>\n AsyncSayManyHellos(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq, void* tag)\n {\n return std::unique_ptr<\n ::grpc::ClientAsyncReaderInterface>>(\n AsyncSayManyHellosRaw(context, request, cq, tag));\n }\n std::unique_ptr<::grpc::ClientAsyncReaderInterface>>\n PrepareAsyncSayManyHellos(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq)\n {\n return std::unique_ptr<\n ::grpc::ClientAsyncReaderInterface>>(\n PrepareAsyncSayManyHellosRaw(context, request, cq));\n }\n\n private:\n virtual ::grpc::ClientAsyncResponseReaderInterface>*\n AsyncSayHelloRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq) = 0;\n virtual ::grpc::ClientAsyncResponseReaderInterface>*\n PrepareAsyncSayHelloRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq) = 0;\n virtual ::grpc::ClientReaderInterface>*\n SayManyHellosRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request) = 0;\n virtual ::grpc::ClientAsyncReaderInterface>*\n AsyncSayManyHellosRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq, void* tag) = 0;\n virtual ::grpc::ClientAsyncReaderInterface>*\n PrepareAsyncSayManyHellosRaw(\n ::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq) = 0;\n };\n class Stub final : public StubInterface\n {\n public:\n Stub(const std::shared_ptr<::grpc::ChannelInterface>& channel);\n ::grpc::Status SayHello(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n flatbuffers::grpc::Message* response) override;\n std::unique_ptr<::grpc::ClientAsyncResponseReader>>\n AsyncSayHello(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq)\n {\n return std::unique_ptr<\n ::grpc::ClientAsyncResponseReader>>(\n AsyncSayHelloRaw(context, request, cq));\n }\n std::unique_ptr<::grpc::ClientAsyncResponseReader>>\n PrepareAsyncSayHello(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq)\n {\n return std::unique_ptr<\n ::grpc::ClientAsyncResponseReader>>(\n PrepareAsyncSayHelloRaw(context, request, cq));\n }\n std::unique_ptr<::grpc::ClientReader>>\n SayManyHellos(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request)\n {\n return std::unique_ptr<::grpc::ClientReader>>(\n SayManyHellosRaw(context, request));\n }\n std::unique_ptr<::grpc::ClientAsyncReader>>\n AsyncSayManyHellos(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq, void* tag)\n {\n return std::unique_ptr<\n ::grpc::ClientAsyncReader>>(\n AsyncSayManyHellosRaw(context, request, cq, tag));\n }\n std::unique_ptr<::grpc::ClientAsyncReader>>\n PrepareAsyncSayManyHellos(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq)\n {\n return std::unique_ptr<\n ::grpc::ClientAsyncReader>>(\n PrepareAsyncSayManyHellosRaw(context, request, cq));\n }\n\n private:\n std::shared_ptr<::grpc::ChannelInterface> channel_;\n ::grpc::ClientAsyncResponseReader>*\n AsyncSayHelloRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq) override;\n ::grpc::ClientAsyncResponseReader>*\n PrepareAsyncSayHelloRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq) override;\n ::grpc::ClientReader>*\n SayManyHellosRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request) override;\n ::grpc::ClientAsyncReader>*\n AsyncSayManyHellosRaw(::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq, void* tag) override;\n ::grpc::ClientAsyncReader>*\n PrepareAsyncSayManyHellosRaw(\n ::grpc::ClientContext* context,\n const flatbuffers::grpc::Message& request,\n ::grpc::CompletionQueue* cq) override;\n const ::grpc::internal::RpcMethod rpcmethod_SayHello_;\n const ::grpc::internal::RpcMethod rpcmethod_SayManyHellos_;\n };\n static std::unique_ptr\n NewStub(const std::shared_ptr<::grpc::ChannelInterface>& channel,\n const ::grpc::StubOptions& options = ::grpc::StubOptions());\n\n class Service : public ::grpc::Service\n {\n public:\n Service();\n virtual ~Service();\n virtual ::grpc::Status SayHello(::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n flatbuffers::grpc::Message* response);\n virtual ::grpc::Status\n SayManyHellos(::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n ::grpc::ServerWriter>* writer);\n };\n template\n class WithAsyncMethod_SayHello : public BaseClass\n {\n private:\n void BaseClassMustBeDerivedFromService(const Service* service) {}\n\n public:\n WithAsyncMethod_SayHello() { ::grpc::Service::MarkMethodAsync(0); }\n ~WithAsyncMethod_SayHello() override { BaseClassMustBeDerivedFromService(this); }\n // disable synchronous version of this method\n ::grpc::Status SayHello(::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n flatbuffers::grpc::Message* response) final override\n {\n abort();\n return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n }\n void RequestSayHello(\n ::grpc::ServerContext* context, flatbuffers::grpc::Message* request,\n ::grpc::ServerAsyncResponseWriter>* response,\n ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq,\n void* tag)\n {\n ::grpc::Service::RequestAsyncUnary(0, context, request, response, new_call_cq,\n notification_cq, tag);\n }\n };\n template\n class WithAsyncMethod_SayManyHellos : public BaseClass\n {\n private:\n void BaseClassMustBeDerivedFromService(const Service* service) {}\n\n public:\n WithAsyncMethod_SayManyHellos() { ::grpc::Service::MarkMethodAsync(1); }\n ~WithAsyncMethod_SayManyHellos() override { BaseClassMustBeDerivedFromService(this); }\n // disable synchronous version of this method\n ::grpc::Status SayManyHellos(\n ::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n ::grpc::ServerWriter>* writer) final override\n {\n abort();\n return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n }\n void RequestSayManyHellos(\n ::grpc::ServerContext* context, flatbuffers::grpc::Message* request,\n ::grpc::ServerAsyncWriter>* writer,\n ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq,\n void* tag)\n {\n ::grpc::Service::RequestAsyncServerStreaming(1, context, request, writer, new_call_cq,\n notification_cq, tag);\n }\n };\n typedef WithAsyncMethod_SayHello> AsyncService;\n template\n class WithGenericMethod_SayHello : public BaseClass\n {\n private:\n void BaseClassMustBeDerivedFromService(const Service* service) {}\n\n public:\n WithGenericMethod_SayHello() { ::grpc::Service::MarkMethodGeneric(0); }\n ~WithGenericMethod_SayHello() override { BaseClassMustBeDerivedFromService(this); }\n // disable synchronous version of this method\n ::grpc::Status SayHello(::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n flatbuffers::grpc::Message* response) final override\n {\n abort();\n return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n }\n };\n template\n class WithGenericMethod_SayManyHellos : public BaseClass\n {\n private:\n void BaseClassMustBeDerivedFromService(const Service* service) {}\n\n public:\n WithGenericMethod_SayManyHellos() { ::grpc::Service::MarkMethodGeneric(1); }\n ~WithGenericMethod_SayManyHellos() override { BaseClassMustBeDerivedFromService(this); }\n // disable synchronous version of this method\n ::grpc::Status SayManyHellos(\n ::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n ::grpc::ServerWriter>* writer) final override\n {\n abort();\n return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n }\n };\n template\n class WithStreamedUnaryMethod_SayHello : public BaseClass\n {\n private:\n void BaseClassMustBeDerivedFromService(const Service* service) {}\n\n public:\n WithStreamedUnaryMethod_SayHello()\n {\n ::grpc::Service::MarkMethodStreamed(\n 0,\n new ::grpc::internal::StreamedUnaryHandler,\n flatbuffers::grpc::Message>(\n std::bind(&WithStreamedUnaryMethod_SayHello::StreamedSayHello, this,\n std::placeholders::_1, std::placeholders::_2)));\n }\n ~WithStreamedUnaryMethod_SayHello() override { BaseClassMustBeDerivedFromService(this); }\n // disable regular version of this method\n ::grpc::Status SayHello(::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n flatbuffers::grpc::Message* response) final override\n {\n abort();\n return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n }\n // replace default version of method with streamed unary\n virtual ::grpc::Status\n StreamedSayHello(::grpc::ServerContext* context,\n ::grpc::ServerUnaryStreamer,\n flatbuffers::grpc::Message>*\n server_unary_streamer) = 0;\n };\n typedef WithStreamedUnaryMethod_SayHello StreamedUnaryService;\n template\n class WithSplitStreamingMethod_SayManyHellos : public BaseClass\n {\n private:\n void BaseClassMustBeDerivedFromService(const Service* service) {}\n\n public:\n WithSplitStreamingMethod_SayManyHellos()\n {\n ::grpc::Service::MarkMethodStreamed(\n 1, new ::grpc::internal::SplitServerStreamingHandler<\n flatbuffers::grpc::Message,\n flatbuffers::grpc::Message>(std::bind(\n &WithSplitStreamingMethod_SayManyHellos::StreamedSayManyHellos,\n this, std::placeholders::_1, std::placeholders::_2)));\n }\n ~WithSplitStreamingMethod_SayManyHellos() override\n {\n BaseClassMustBeDerivedFromService(this);\n }\n // disable regular version of this method\n ::grpc::Status SayManyHellos(\n ::grpc::ServerContext* context,\n const flatbuffers::grpc::Message* request,\n ::grpc::ServerWriter>* writer) final override\n {\n abort();\n return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n }\n // replace default version of method with split streamed\n virtual ::grpc::Status StreamedSayManyHellos(\n ::grpc::ServerContext* context,\n ::grpc::ServerSplitStreamer,\n flatbuffers::grpc::Message>*\n server_split_streamer) = 0;\n };\n typedef WithSplitStreamingMethod_SayManyHellos SplitStreamedService;\n typedef WithStreamedUnaryMethod_SayHello>\n StreamedService;\n};\n\n#endif // GRPC_example__INCLUDED\n" }, { "path": "examples/12_FlatBuffers/example_generated.h", "content": "// automatically generated by the FlatBuffers compiler, do not modify\n\n#ifndef FLATBUFFERS_GENERATED_EXAMPLE_H_\n#define FLATBUFFERS_GENERATED_EXAMPLE_H_\n\n#include \"flatbuffers/flatbuffers.h\"\n\nstruct HelloReply;\n\nstruct HelloRequest;\n\nstruct ManyHellosRequest;\n\nstruct HelloReply FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table\n{\n enum\n {\n VT_MESSAGE = 4\n };\n const flatbuffers::String* message() const\n {\n return GetPointer(VT_MESSAGE);\n }\n bool Verify(flatbuffers::Verifier& verifier) const\n {\n return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_MESSAGE) &&\n verifier.VerifyString(message()) && verifier.EndTable();\n }\n};\n\nstruct HelloReplyBuilder\n{\n flatbuffers::FlatBufferBuilder& fbb_;\n flatbuffers::uoffset_t start_;\n void add_message(flatbuffers::Offset message)\n {\n fbb_.AddOffset(HelloReply::VT_MESSAGE, message);\n }\n explicit HelloReplyBuilder(flatbuffers::FlatBufferBuilder& _fbb) : fbb_(_fbb)\n {\n start_ = fbb_.StartTable();\n }\n HelloReplyBuilder& operator=(const HelloReplyBuilder&);\n flatbuffers::Offset Finish()\n {\n const auto end = fbb_.EndTable(start_);\n auto o = flatbuffers::Offset(end);\n return o;\n }\n};\n\ninline flatbuffers::Offset\n CreateHelloReply(flatbuffers::FlatBufferBuilder& _fbb,\n flatbuffers::Offset message = 0)\n{\n HelloReplyBuilder builder_(_fbb);\n builder_.add_message(message);\n return builder_.Finish();\n}\n\ninline flatbuffers::Offset CreateHelloReplyDirect(flatbuffers::FlatBufferBuilder& _fbb,\n const char* message = nullptr)\n{\n return CreateHelloReply(_fbb, message ? _fbb.CreateString(message) : 0);\n}\n\nstruct HelloRequest FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table\n{\n enum\n {\n VT_NAME = 4\n };\n const flatbuffers::String* name() const\n {\n return GetPointer(VT_NAME);\n }\n bool Verify(flatbuffers::Verifier& verifier) const\n {\n return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_NAME) &&\n verifier.VerifyString(name()) && verifier.EndTable();\n }\n};\n\nstruct HelloRequestBuilder\n{\n flatbuffers::FlatBufferBuilder& fbb_;\n flatbuffers::uoffset_t start_;\n void add_name(flatbuffers::Offset name)\n {\n fbb_.AddOffset(HelloRequest::VT_NAME, name);\n }\n explicit HelloRequestBuilder(flatbuffers::FlatBufferBuilder& _fbb) : fbb_(_fbb)\n {\n start_ = fbb_.StartTable();\n }\n HelloRequestBuilder& operator=(const HelloRequestBuilder&);\n flatbuffers::Offset Finish()\n {\n const auto end = fbb_.EndTable(start_);\n auto o = flatbuffers::Offset(end);\n return o;\n }\n};\n\ninline flatbuffers::Offset\n CreateHelloRequest(flatbuffers::FlatBufferBuilder& _fbb,\n flatbuffers::Offset name = 0)\n{\n HelloRequestBuilder builder_(_fbb);\n builder_.add_name(name);\n return builder_.Finish();\n}\n\ninline flatbuffers::Offset\n CreateHelloRequestDirect(flatbuffers::FlatBufferBuilder& _fbb, const char* name = nullptr)\n{\n return CreateHelloRequest(_fbb, name ? _fbb.CreateString(name) : 0);\n}\n\nstruct ManyHellosRequest FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table\n{\n enum\n {\n VT_NAME = 4,\n VT_NUM_GREETINGS = 6\n };\n const flatbuffers::String* name() const\n {\n return GetPointer(VT_NAME);\n }\n int32_t num_greetings() const { return GetField(VT_NUM_GREETINGS, 0); }\n bool Verify(flatbuffers::Verifier& verifier) const\n {\n return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_NAME) &&\n verifier.VerifyString(name()) && VerifyField(verifier, VT_NUM_GREETINGS) &&\n verifier.EndTable();\n }\n};\n\nstruct ManyHellosRequestBuilder\n{\n flatbuffers::FlatBufferBuilder& fbb_;\n flatbuffers::uoffset_t start_;\n void add_name(flatbuffers::Offset name)\n {\n fbb_.AddOffset(ManyHellosRequest::VT_NAME, name);\n }\n void add_num_greetings(int32_t num_greetings)\n {\n fbb_.AddElement(ManyHellosRequest::VT_NUM_GREETINGS, num_greetings, 0);\n }\n explicit ManyHellosRequestBuilder(flatbuffers::FlatBufferBuilder& _fbb) : fbb_(_fbb)\n {\n start_ = fbb_.StartTable();\n }\n ManyHellosRequestBuilder& operator=(const ManyHellosRequestBuilder&);\n flatbuffers::Offset Finish()\n {\n const auto end = fbb_.EndTable(start_);\n auto o = flatbuffers::Offset(end);\n return o;\n }\n};\n\ninline flatbuffers::Offset\n CreateManyHellosRequest(flatbuffers::FlatBufferBuilder& _fbb,\n flatbuffers::Offset name = 0,\n int32_t num_greetings = 0)\n{\n ManyHellosRequestBuilder builder_(_fbb);\n builder_.add_num_greetings(num_greetings);\n builder_.add_name(name);\n return builder_.Finish();\n}\n\ninline flatbuffers::Offset\n CreateManyHellosRequestDirect(flatbuffers::FlatBufferBuilder& _fbb, const char* name = nullptr,\n int32_t num_greetings = 0)\n{\n return CreateManyHellosRequest(_fbb, name ? _fbb.CreateString(name) : 0, num_greetings);\n}\n\n#endif // FLATBUFFERS_GENERATED_EXAMPLE_H_\n" }, { "path": "examples/12_FlatBuffers/server.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n#include \"tensorrt/laboratory/core/resources.h\"\n\n#include \"example.grpc.fb.h\"\n#include \"example_generated.h\"\n\nusing nvrpc::AsyncRPC;\nusing nvrpc::AsyncService;\nusing nvrpc::Context;\nusing nvrpc::Executor;\nusing nvrpc::Server;\nusing trtlab::Resources;\n\nusing Request = flatbuffers::grpc::Message;\nusing Response = flatbuffers::grpc::Message;\n\nstruct SimpleResources : public Resources\n{\n};\n\nclass SimpleContext final : public Context\n{\n void ExecuteRPC(Request& input, Response& output) final override\n {\n flatbuffers::grpc::MessageBuilder mb_;\n\n // We call GetRoot to \"parse\" the message. Verification is already\n // performed by default. See the notes below for more details.\n const HelloRequest* request = input.GetRoot();\n\n // Fields are retrieved as usual with FlatBuffers\n const std::string& name = request->name()->str();\n\n // `flatbuffers::grpc::MessageBuilder` is a `FlatBufferBuilder` with a\n // special allocator for efficient gRPC buffer transfer, but otherwise\n // usage is the same as usual.\n auto msg_offset = mb_.CreateString(\"Hello, \" + name);\n auto hello_offset = CreateHelloReply(mb_, msg_offset);\n mb_.Finish(hello_offset);\n\n // The `ReleaseMessage()` function detaches the message from the\n // builder, so we can transfer the resopnse to gRPC while simultaneously\n // detaching that memory buffer from the builer.\n output = mb_.ReleaseMessage();\n CHECK(output.Verify());\n this->FinishResponse();\n }\n};\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n\n ::google::InitGoogleLogging(\"flatbuffer service\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n // A server will bind an IP:PORT to listen on\n Server server(\"0.0.0.0:50051\");\n\n // A server can host multiple services\n auto simpleInference = server.RegisterAsyncService();\n\n auto rpcCompute =\n simpleInference->RegisterRPC(&Greeter::AsyncService::RequestSayHello);\n\n auto rpcResources = std::make_shared();\n auto executor = server.RegisterExecutor(new Executor(1));\n executor->RegisterContexts(rpcCompute, rpcResources, 10);\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(2000), [] {\n // This is a timeout loop executed every 2seconds\n // Run() with no arguments will run an empty timeout loop every 5 seconds.\n // RunAsync() will return immediately, its your responsibility to ensure the\n // server doesn't go out of scope or a Shutdown will be triggered on your services.\n });\n}\n" }, { "path": "examples/30_PyTensorRT/README.md", "content": "# Python Inference Example\n\n\n" }, { "path": "examples/30_PyTensorRT/client.py", "content": "#!/usr/bin/env python3\nimport os\nimport time\n\nimport numpy as np\n\nimport infer\nimport infer_test_utils as utils\n\n\ndef main():\n manager = infer.RemoteInferenceManager(hostname=\"localhost:50052\")\n models = manager.get_models()\n print(models)\n\n mnist = manager.infer_runner(\"mnist\")\n\n print(\"Input Bindings: {}\".format(mnist.input_bindings()))\n print(\"Output Bindings: {}\".format(mnist.output_bindings()))\n\n inputs = utils.load_inputs(\"/work/models/onnx/mnist-v1.3/test_data_set_0\")\n expected = utils.load_outputs(\"/work/models/onnx/mnist-v1.3/test_data_set_0\")\n\n start = time.process_time()\n results = [mnist.infer(Input3=input) for input in inputs]\n results = [r.get() for r in results]\n print(\"Compute Time: {}\".format(time.process_time() - start))\n print(results)\n\n# for r, e in zip(results, expected):\n# for key, val in r.items():\n# print(\"Output Binding Name: {}; shape{}\".format(key, val.shape))\n# r = val.reshape((1,10))\n# np.testing.assert_almost_equal(r, e, decimal=3) \n\n# models.serve()\n #mnist_model = models.get_model(\"mnist\")\n #benchmark = infer.InferBench(models)\n #benchmark.run(mnist_model, 1, 0.1)\n #print(results)\n \n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "examples/30_PyTensorRT/compute.py", "content": "#!/usr/bin/env python3\nimport itertools\nimport os\nimport time\n\nimport numpy as np\n\nimport trtlab\nimport infer_test_utils as utils\n\n\ndef main():\n models = trtlab.InferenceManager(max_exec_concurrency=1)\n mnist = models.register_tensorrt_engine(\"mnist\", \"/work/models/onnx/mnist-v1.3/mnist-v1.3.engine\")\n models.update_resources()\n\n print(\"Input Bindings: {}\".format(mnist.input_bindings()))\n print(\"Output Bindings: {}\".format(mnist.output_bindings()))\n\n inputs = utils.load_inputs(\"/work/models/onnx/mnist-v1.3/test_data_set_0\")\n expected = utils.load_outputs(\"/work/models/onnx/mnist-v1.3/test_data_set_0\")\n\n start = time.process_time()\n while True:\n futures = [mnist.infer(Input3=inputs[0]) for _ in range(100)]\n results = [f.get() for f in futures]\n# while True:\n# results = [mnist.infer(Input3=input) for input in itertools.repeat(inputs[0], 1000)]\n# results = [r.get() for r in results]\n# time.sleep(0.1)\n print(\"Compute Time: {}\".format(time.process_time() - start))\n \n# for r, e in zip(results, expected):\n# for key, val in r.items():\n# print(\"Output Binding Name: {}; shape{}\".format(key, val.shape))\n# r = val.reshape((1,10))\n# np.testing.assert_almost_equal(r, e, decimal=3) \n\n #mnist_model = models.get_model(\"mnist\")\n #benchmark = infer.InferBench(models)\n #benchmark.run(mnist_model, 1, 0.1)\n #print(results)\n \n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "examples/30_PyTensorRT/infer_test_utils.py", "content": "#!/usr/bin/env python3\nimport glob\nimport os\n\nimport onnx\nfrom onnx import numpy_helper\nfrom matplotlib import pyplot as plt\nimport numpy as np\n\ndef load_inputs(test_data_dir):\n # Load inputs\n inputs = []\n inputs_num = len(glob.glob(os.path.join(test_data_dir, 'input_*.pb')))\n for i in range(inputs_num):\n input_file = os.path.join(test_data_dir, 'input_{}.pb'.format(i))\n tensor = onnx.TensorProto()\n with open(input_file, 'rb') as f:\n tensor.ParseFromString(f.read())\n inputs.append(numpy_helper.to_array(tensor))\n return inputs\n\ndef load_outputs(test_data_dir):\n # Load reference outputs\n ref_outputs = []\n ref_outputs_num = len(glob.glob(os.path.join(test_data_dir, 'output_*.pb')))\n for i in range(ref_outputs_num):\n output_file = os.path.join(test_data_dir, 'output_{}.pb'.format(i))\n tensor = onnx.TensorProto()\n with open(output_file, 'rb') as f:\n tensor.ParseFromString(f.read())\n ref_outputs.append(numpy_helper.to_array(tensor))\n return ref_outputs\n\ndef mnist_image(data):\n two_d = (np.reshape(data, (28, 28))).astype(np.uint8)\n plt.imshow(two_d, interpolation='nearest')\n return plt\n\ndef softmax(x):\n \"\"\"Compute softmax values for each sets of scores in x.\"\"\"\n e_x = np.exp(x - np.max(x))\n return e_x / e_x.sum()\n" }, { "path": "examples/30_PyTensorRT/rebuild.sh", "content": "#!/bin/bash\ncd /work/build/tensorrt-laboratory/python\nmake -j\ncd /work/examples/30_PyTensorRT\nif [ ! -e infer.cpython-35m-x86_64-linux-gnu.so ]; then\n ln -s /work/build/tensorrt-laboratory/python/tensorrt/infer.cpython-35m-x86_64-linux-gnu.so\nfi\n" }, { "path": "examples/30_PyTensorRT/server.py", "content": "#!/usr/bin/env python3\nimport os\nimport time\n\nimport numpy as np\n\nimport infer\nimport infer_test_utils as utils\n\n\ndef main():\n models = infer.InferenceManager(max_exec_concurrency=2)\n mnist = models.register_tensorrt_engine(\"mnist\", \"/work/models/onnx/mnist-v1.3/mnist-v1.3.engine\")\n models.update_resources()\n\n print(\"Input Bindings: {}\".format(mnist.input_bindings()))\n print(\"Output Bindings: {}\".format(mnist.output_bindings()))\n\n inputs = utils.load_inputs(\"/work/models/onnx/mnist-v1.3/test_data_set_0\")\n expected = utils.load_outputs(\"/work/models/onnx/mnist-v1.3/test_data_set_0\")\n\n start = time.process_time()\n results = [mnist.infer(Input3=input) for input in inputs]\n results = [r.get() for r in results]\n print(\"Compute Time: {}\".format(time.process_time() - start))\n\n for r, e in zip(results, expected):\n for key, val in r.items():\n print(\"Output Binding Name: {}; shape{}\".format(key, val.shape))\n r = val.reshape((1,10))\n np.testing.assert_almost_equal(r, e, decimal=3) \n\n models.serve()\n #mnist_model = models.get_model(\"mnist\")\n #benchmark = infer.InferBench(models)\n #benchmark.run(mnist_model, 1, 0.1)\n #print(results)\n \n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "examples/90_Kubernetes/README.md", "content": "# Kubernetes\n\nUsing [Kubernetes on NVIDIA GPUs, aka KONG](https://developer.nvidia.com/kubernetes-gpu) is a great\nway of deploying GPU accelerated microservices. This page will act as a guide for\nfor both development and production deployment.\n\n* For development, we will use [minikube](https://kubernetes.io/docs/setup/minikube/) to deploy a single-node\nKubernetes cluster.\n* For production, we will use a Kubernetes cluster installed by the \n[DeepOps project](https://github.com/nvidia/deepops).\n\n## Prerequisites\n\n* [Kubernetes v1.10.0](https://kubernetes.io) \n* [NVIDIA GPU Device Plugin](https://github.com/NVIDIA/k8s-device-plugin#preparing-your-gpu-nodes)\n* [Helm](https://helm.sh)\n* [prometheus-operator](https://github.com/coreos/prometheus-operator)\n ```\n helm repo add coreos https://s3-eu-west-1.amazonaws.com/coreos-charts/stable/\n ```\n\n## Setup\n\nThe following packages will be installed on your Kubernetes cluster:\n* [CoreOS's Prometheus Operator](https://github.com/coreos/prometheus-operator) for gathering and monitoring metrics\n* [Istio v0.8](https://istio.io) for ingress and load-balancing\n\nAfter the installation of those packages, we will deploy the following:\n* Scalable [K8s Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/)\nof the [TensorRT GRPC example](../02_TensorRT_GRPC) `inference-grpc.x` \n* YAIS specific instance of a Prometheus server that will scrape any Pods labeled `scrape: yais`\n* Istio `Gateway` and `VirtualService` to route load-balanced traffic to our gRPC service.\n\n## Install\n\nStarting at this point, you should have a Kubernetes cluter with all the prerequisites.\n\nIf you use the [minikube setup](minikube/README.md) you can simply run:\n```\n./bootstrap-minikube.sh\n```\n\nOtherwise, you can choose to install each of the components manually.\n\n### Prometheus Operator\n\nInitialize Helm and install the `prometheus-operator` and `kube-prometheus`\n\n```\ncd ../prometheus\n./bootstrap.sh\ncd ..\n```\n\nMonitor `kubectl get pods -n monitoring` and wait everything to come up.\n\nCustomize any settings in the [custom-settings.yml](prometheus/custom-settings.yml)\nfile. This project is exposing the Grafana server as a `NodePort` and providing custom \ndatasource and dashboards for YAIS metrics.\n\n### Istio\n\nInitialize Istio. I've provided the Istio v0.8 `istio-demo.yml` modified to use a `NodePort`\nas `istio/minikube.yml` If you are using a cloude instance, you can change to a `LoadBalancer`.\n\n```\nkubectl create namespace istio-system\nkubectl apply -f istio/istio-v1.0-minikube.yml\nkubectl label namespace default istio-injection=enabled\n```\n\n### YAIS Service\n\n```\nkubectl apply -f yais-deploy.yml\n```\n\nThis does the following:\n* `Deployment` - launches the service and resources\n* `Service` - provides access policy to the deployment pods\n* `ServiceMonitor` - tells our Prometheus server to scrape YAIS metrics\n* `Gateway` - ingress host, port and protocol\n* `VirtualService - routing ingress to services\n\n### Test the Service\n\nUse the [`devel.sh`](devel.sh) script in the project's root directory.\n\n```\n# from project root\n./devel.sh\ncd build/examples/02_TensorRT_GRPC\n./siege.x --port 31380 --rate=1000\n```\n\n`31380` is the default `NodePort` for the Istio `ingressgateway`.\n\nNote: If you get errors, sometime it takes a short period of time before the ingress gateway\nis updated to reflect the routing. \n\n### Check the Metrics\n\n```\nkubectl get svc -n monitoring | grep grafana\n```\n\nThe default login is `admin/admin`. Navigate to the `YAIS` dashboard. Celebrate.\n" }, { "path": "examples/90_Kubernetes/bootstrap-minikube.sh", "content": "#!/bin/bash\n\nif ! [ -x \"$(command -v helm)\" ]; then\n echo 'Error: helm is not installed.' >&2\n exit 1\nfi\n\n# minikube\n(cd minikube && ./bootstrap.sh)\n\n# prometheus-operator\n(cd prometheus && ./bootstrap.sh)\n\n# istio\nkubectl create namespace istio-system\nkubectl apply -f istio/istio-v1.0-minikube.yml\n\nsleep 30\nkubectl label namespace default istio-injection=enabled\n\n# deploy yais example\nkubectl apply -f yais-deploy.yml\n\n# sleep 15\n# kubectl label namespace default istio-injection-\n" }, { "path": "examples/90_Kubernetes/deploy/build-and-run.sh", "content": "#!/bin/bash\n\ndefault_engine=/work/models/ResNet-152-b8-fp16.engine\nconcurrency=${YAIS_CONCURRENCY:-1}\nengine=${YAIS_TRT_ENGINE:-$default_engine}\n\nif [ \"$engine\" = \"$default_engine\" ]; then\n if [ ! -e $engine ]; then\n cd /work/models\n ./setup.py\n fi\nfi\n\n/work/build/examples/02_TensorRT_GRPC/inference-grpc.x \\\n --engine=${engine} \\\n --contexts=${concurrency}\n" }, { "path": "examples/90_Kubernetes/devel/README.md", "content": "\n## Round 1: External Service\n\nBefore deploying a YAIS service with Kubernetes, we will first setup a developer environment\nwhere we will execute our service in a Docker development container. We can still use our\nKubernetes/Prometheus/Grafana environment to gather and visualize metrics. To do so, we\nwill create an external service pointing at our host.\n\nEdit the `yais-devel.yml` and modify the IP address of the `Endpoint` to point at the host machine running\nminikube (`sudo minikube ip`). \n\n```\napiVersion: v1\nkind: Endpoints\nmetadata:\n name: yais-devel\nsubsets:\n- addresses:\n - ip: 10.0.0.10 # <== ChangeMe\n ports:\n - name: metrics\n port: 50078\n```\n\n```\nkubectl apply -f yais-devel.yml\n```\n\nThis will create a Prometheus `ServiceMonitor` that will scrape from the external service, i.e. the \nDocker development container. This is a good first start at integrating your service into Kubernetes\nwithout having to do a full blown deployment.\n\nCongrats, your minikube cluster now looking for services labeled `scrape: yais` and if found, will\nautomatically start scraped the port labels `metrics`. \n\nThe final step is to bring an inference service online and to generate some load on that service.\nLaunch the YAIS developement container using the `devel.sh` script in the project's root directory.\nMake sure all the examples have been built and models have been build, \nsee [README::Quickstart](README.md#quickstart). \n\n```\ncd examples/97_SingleProcessMultiSteam\n\nroot@dgx:/work/examples/97_SingleProcessMultiSteam# ./launch_service.sh 1 1 /work/models/ResNet-50-b1-fp32.engine\nI0709 10:13:41.175212 468 Server.cc:37] gRPC listening on: 0.0.0.0:50051\nI0709 10:13:41.175477 468 server.cc:229] Register Service (flowers::Inference) with Server\nI0709 10:13:41.175492 468 server.cc:238] Register RPC (flowers::Inference::Compute) with Service (flowers::Inference)\nI0709 10:13:41.175500 468 server.cc:243] Initializing Resources for RPC (flowers::Inference::Compute)\nI0709 10:13:41.273568 468 TensorRT.cc:561] -- Initialzing TensorRT Resource Manager --\nI0709 10:13:41.273602 468 TensorRT.cc:562] Maximum Execution Concurrency: 1\nI0709 10:13:41.273609 468 TensorRT.cc:563] Maximum Copy Concurrency: 3\nI0709 10:13:42.596443 468 TensorRT.cc:628] -- Registering Model: flowers --\nI0709 10:13:42.596500 468 TensorRT.cc:629] Input/Output Tensors require 591.9 KiB\nI0709 10:13:42.596511 468 TensorRT.cc:630] Execution Activations require 7.8 MiB\nI0709 10:13:42.604210 468 TensorRT.cc:652] -- Allocating TensorRT Resources --\nI0709 10:13:42.604228 468 TensorRT.cc:653] Creating 1 TensorRT execution tokens.\nI0709 10:13:42.604236 468 TensorRT.cc:654] Creating a Pool of 3 Host/Device Memory Stacks\nI0709 10:13:42.604248 468 TensorRT.cc:655] Each Host Stack contains 608.0 KiB\nI0709 10:13:42.604256 468 TensorRT.cc:656] Each Device Stack contains 8.5 MiB\nI0709 10:13:42.604264 468 TensorRT.cc:657] Total GPU Memory: 25.5 MiB\nI0709 10:13:42.606546 468 server.cc:255] Initializing Executor\nI0709 10:13:42.606832 468 server.cc:259] Registering Execution Contexts for RPC (flowers::Inference::Compute) with Executor\nI0709 10:13:42.606889 468 server.cc:262] Running Server\n\nwarmup with client-async.x\n1000 requests in 2.60522seconds; inf/sec: 383.845\n\nStarting a shell keeping the services and load-balancer running...\nTry /work/build/examples/02_TensorRT_GRPC/siege.x --rate=2000 --port=50051\n\n1 x /work/models/ResNet-50-b1-fp32.engine Subshell:\n```\n\nUse `telegraf` and watch the scrape count; the `yais-devel` scraper is set to pull metrics every 2 seconds.\nIt can up to a minute or so until you see scraping from your k8s cluster.\n\n```\n1 x /work/models/ResNet-50-b1-fp32.engine Subshell: telegraf -test -config /work/examples/91_Prometheus/scrape.conf\n...\n> exposer_bytes_transferred,host=dgx,url=http://localhost:50078/metrics counter=0 1531131559000000000 # <== watch the counter\n...\n```\n\n## Round 2: Package and Deploy\n\nTODO - We could use some community help here.\n\n## Round 3: Optimize Deploy Contaienr\n\nTODO - We could use some community help here.\n" }, { "path": "examples/90_Kubernetes/devel/yais-devel.yml", "content": "---\napiVersion: v1\nkind: Service\nmetadata:\n name: yais-devel\n labels:\n app: yais-devel\nspec:\n ports:\n - name: metrics\n port: 51078\n targetPort: 50078\n---\napiVersion: v1\nkind: Endpoints\nmetadata:\n name: yais-devel\nsubsets:\n- addresses:\n - ip: 10.0.0.10\n ports:\n - name: metrics\n port: 50078\n---\n# this will get scraped by the default kube-prometheus\n# and the yais-metric prometheus service (if it is running)\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n name: yais-devel\n labels:\n scrape: yais\nspec:\n selector:\n matchLabels:\n app: yais-devel\n endpoints:\n - port: metrics\n interval: 2s\n honorLabels: true\n" }, { "path": "examples/90_Kubernetes/istio/README.md", "content": "# Istio\n\n## Install\n\n```\n# Download the latest release\ncurl -L https://git.io/getLatestIstio | sh -\n\n# Istio 1.0\nhelm template install/kubernetes/helm/istio --name istio --namespace istio-system \\\n --set gateways.istio-ingressgateway.type=NodePort \\\n --set gateways.istio-egressgateway.type=NodePort > istio-v1.0-minikube.yml\n```\n\nInstall Istio and enable the default namespace for injection; however, only\npods with the proper annotations will have sidecars injected.\n\n```\nkubectl create namespace istio-system\nkubectl apply -f istio-v1.0-minikube.yml\nkubectl label namespace default istio-injection=enabled\nkubectl get namespace -L istio-injection\n```\n\nThe annotation required for sidecar injection:\n```\napiVersion: extensions/v1beta1\nkind: Deployment\n...\nspec:\n template:\n metadata:\n annotations: # <== sidecar\n sidecar.istio.io/inject: \"false\" # <== annotation\n...\n```\n" }, { "path": "examples/90_Kubernetes/istio/rendered/istio-v0.8-minikube.yml", "content": "apiVersion: v1\nkind: Namespace\nmetadata:\n name: istio-system\n---\n# Source: istio/charts/mixer/templates/configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio-statsd-prom-bridge\n namespace: istio-system\n labels:\n app: istio-statsd-prom-bridge\n chart: mixer-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: mixer\ndata:\n mapping.conf: |-\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio-mixer-custom-resources\n namespace: istio-system\n labels:\n app: istio-mixer\n chart: mixer-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: mixer\ndata:\n custom-resources.yaml: |- \n apiVersion: \"config.istio.io/v1alpha2\"\n kind: attributemanifest\n metadata:\n name: istioproxy\n namespace: istio-system\n spec:\n attributes:\n origin.ip:\n valueType: IP_ADDRESS\n origin.uid:\n valueType: STRING\n origin.user:\n valueType: STRING\n request.headers:\n valueType: STRING_MAP\n request.id:\n valueType: STRING\n request.host:\n valueType: STRING\n request.method:\n valueType: STRING\n request.path:\n valueType: STRING\n request.reason:\n valueType: STRING\n request.referer:\n valueType: STRING\n request.scheme:\n valueType: STRING\n request.total_size:\n valueType: INT64\n request.size:\n valueType: INT64\n request.time:\n valueType: TIMESTAMP\n request.useragent:\n valueType: STRING\n response.code:\n valueType: INT64\n response.duration:\n valueType: DURATION\n response.headers:\n valueType: STRING_MAP\n response.total_size:\n valueType: INT64\n response.size:\n valueType: INT64\n response.time:\n valueType: TIMESTAMP\n source.uid:\n valueType: STRING\n source.user:\n valueType: STRING\n destination.uid:\n valueType: STRING\n connection.id:\n valueType: STRING\n connection.received.bytes:\n valueType: INT64\n connection.received.bytes_total:\n valueType: INT64\n connection.sent.bytes:\n valueType: INT64\n connection.sent.bytes_total:\n valueType: INT64\n connection.duration:\n valueType: DURATION\n connection.mtls:\n valueType: BOOL\n context.protocol:\n valueType: STRING\n context.timestamp:\n valueType: TIMESTAMP\n context.time:\n valueType: TIMESTAMP\n api.service:\n valueType: STRING\n api.version:\n valueType: STRING\n api.operation:\n valueType: STRING\n api.protocol:\n valueType: STRING\n request.auth.principal:\n valueType: STRING\n request.auth.audiences:\n valueType: STRING\n request.auth.presenter:\n valueType: STRING\n request.auth.claims:\n valueType: STRING_MAP\n request.auth.raw_claims:\n valueType: STRING\n request.api_key:\n valueType: STRING\n \n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: attributemanifest\n metadata:\n name: kubernetes\n namespace: istio-system\n spec:\n attributes:\n source.ip:\n valueType: IP_ADDRESS\n source.labels:\n valueType: STRING_MAP\n source.name:\n valueType: STRING\n source.namespace:\n valueType: STRING\n source.service:\n valueType: STRING\n source.serviceAccount:\n valueType: STRING\n destination.ip:\n valueType: IP_ADDRESS\n destination.labels:\n valueType: STRING_MAP\n destination.name:\n valueType: STRING\n destination.namespace:\n valueType: STRING\n destination.service:\n valueType: STRING\n destination.serviceAccount:\n valueType: STRING\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: stdio\n metadata:\n name: handler\n namespace: istio-system\n spec:\n outputAsJson: true\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: logentry\n metadata:\n name: accesslog\n namespace: istio-system\n spec:\n severity: '\"Info\"'\n timestamp: request.time\n variables:\n originIp: origin.ip | ip(\"0.0.0.0\")\n sourceIp: source.ip | ip(\"0.0.0.0\")\n sourceService: source.service | \"\"\n sourceUser: source.user | source.uid | \"\"\n sourceNamespace: source.namespace | \"\"\n destinationIp: destination.ip | ip(\"0.0.0.0\")\n destinationService: destination.service | \"\"\n destinationNamespace: destination.namespace | \"\"\n apiName: api.service | \"\"\n apiVersion: api.version | \"\"\n apiClaims: request.headers[\"sec-istio-auth-userinfo\"]| \"\"\n apiKey: request.api_key | request.headers[\"x-api-key\"] | \"\"\n requestOperation: api.operation | \"\"\n protocol: request.scheme | \"http\"\n method: request.method | \"\"\n url: request.path | \"\"\n responseCode: response.code | 0\n responseSize: response.size | 0\n requestSize: request.size | 0\n latency: response.duration | \"0ms\"\n connectionMtls: connection.mtls | false\n userAgent: request.useragent | \"\"\n responseTimestamp: response.time\n receivedBytes: request.total_size | connection.received.bytes | 0\n sentBytes: response.total_size | connection.sent.bytes | 0\n referer: request.referer | \"\"\n monitored_resource_type: '\"UNSPECIFIED\"'\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: rule\n metadata:\n name: stdio\n namespace: istio-system\n spec:\n match: \"true\" # If omitted match is true.\n actions:\n - handler: handler.stdio\n instances:\n - accesslog.logentry\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: metric\n metadata:\n name: requestcount\n namespace: istio-system\n spec:\n value: \"1\"\n dimensions:\n source_service: source.service | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_service: destination.service | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n response_code: response.code | 200\n connection_mtls: connection.mtls | false\n monitored_resource_type: '\"UNSPECIFIED\"'\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: metric\n metadata:\n name: requestduration\n namespace: istio-system\n spec:\n value: response.duration | \"0ms\"\n dimensions:\n source_service: source.service | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_service: destination.service | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n response_code: response.code | 200\n connection_mtls: connection.mtls | false\n monitored_resource_type: '\"UNSPECIFIED\"'\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: metric\n metadata:\n name: requestsize\n namespace: istio-system\n spec:\n value: request.size | 0\n dimensions:\n source_service: source.service | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_service: destination.service | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n response_code: response.code | 200\n connection_mtls: connection.mtls | false\n monitored_resource_type: '\"UNSPECIFIED\"'\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: metric\n metadata:\n name: responsesize\n namespace: istio-system\n spec:\n value: response.size | 0\n dimensions:\n source_service: source.service | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_service: destination.service | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n response_code: response.code | 200\n connection_mtls: connection.mtls | false\n monitored_resource_type: '\"UNSPECIFIED\"'\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: metric\n metadata:\n name: tcpbytesent\n namespace: istio-system\n labels:\n istio-protocol: tcp # needed so that mixer will only generate when context.protocol == tcp\n spec:\n value: connection.sent.bytes | 0\n dimensions:\n source_service: source.service | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_service: destination.service | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n connection_mtls: connection.mtls | false\n monitored_resource_type: '\"UNSPECIFIED\"'\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: metric\n metadata:\n name: tcpbytereceived\n namespace: istio-system\n labels:\n istio-protocol: tcp # needed so that mixer will only generate when context.protocol == tcp\n spec:\n value: connection.received.bytes | 0\n dimensions:\n source_service: source.service | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_service: destination.service | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n connection_mtls: connection.mtls | false\n monitored_resource_type: '\"UNSPECIFIED\"'\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: prometheus\n metadata:\n name: handler\n namespace: istio-system\n spec:\n metrics:\n - name: request_count\n instance_name: requestcount.metric.istio-system\n kind: COUNTER\n label_names:\n - source_service\n - source_version\n - destination_service\n - destination_version\n - response_code\n - connection_mtls\n - name: request_duration\n instance_name: requestduration.metric.istio-system\n kind: DISTRIBUTION\n label_names:\n - source_service\n - source_version\n - destination_service\n - destination_version\n - response_code\n - connection_mtls\n buckets:\n explicit_buckets:\n bounds: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]\n - name: request_size\n instance_name: requestsize.metric.istio-system\n kind: DISTRIBUTION\n label_names:\n - source_service\n - source_version\n - destination_service\n - destination_version\n - response_code\n - connection_mtls\n buckets:\n exponentialBuckets:\n numFiniteBuckets: 8\n scale: 1\n growthFactor: 10\n - name: response_size\n instance_name: responsesize.metric.istio-system\n kind: DISTRIBUTION\n label_names:\n - source_service\n - source_version\n - destination_service\n - destination_version\n - response_code\n - connection_mtls\n buckets:\n exponentialBuckets:\n numFiniteBuckets: 8\n scale: 1\n growthFactor: 10\n - name: tcp_bytes_sent\n instance_name: tcpbytesent.metric.istio-system\n kind: COUNTER\n label_names:\n - source_service\n - source_version\n - destination_service\n - destination_version\n - connection_mtls\n - name: tcp_bytes_received\n instance_name: tcpbytereceived.metric.istio-system\n kind: COUNTER\n label_names:\n - source_service\n - source_version\n - destination_service\n - destination_version\n - connection_mtls\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: rule\n metadata:\n name: promhttp\n namespace: istio-system\n labels:\n istio-protocol: http\n spec:\n actions:\n - handler: handler.prometheus\n instances:\n - requestcount.metric\n - requestduration.metric\n - requestsize.metric\n - responsesize.metric\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: rule\n metadata:\n name: promtcp\n namespace: istio-system\n labels:\n istio-protocol: tcp # needed so that mixer will only execute when context.protocol == TCP\n spec:\n actions:\n - handler: handler.prometheus\n instances:\n - tcpbytesent.metric\n - tcpbytereceived.metric\n ---\n \n apiVersion: \"config.istio.io/v1alpha2\"\n kind: kubernetesenv\n metadata:\n name: handler\n namespace: istio-system\n spec:\n # when running from mixer root, use the following config after adding a\n # symbolic link to a kubernetes config file via:\n #\n # $ ln -s ~/.kube/config mixer/adapter/kubernetes/kubeconfig\n #\n # kubeconfig_path: \"mixer/adapter/kubernetes/kubeconfig\"\n \n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: rule\n metadata:\n name: kubeattrgenrulerule\n namespace: istio-system\n spec:\n actions:\n - handler: handler.kubernetesenv\n instances:\n - attributes.kubernetes\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: rule\n metadata:\n name: tcpkubeattrgenrulerule\n namespace: istio-system\n spec:\n match: context.protocol == \"tcp\"\n actions:\n - handler: handler.kubernetesenv\n instances:\n - attributes.kubernetes\n ---\n apiVersion: \"config.istio.io/v1alpha2\"\n kind: kubernetes\n metadata:\n name: attributes\n namespace: istio-system\n spec:\n # Pass the required attribute data to the adapter\n source_uid: source.uid | \"\"\n source_ip: source.ip | ip(\"0.0.0.0\") # default to unspecified ip addr\n destination_uid: destination.uid | \"\"\n origin_uid: '\"\"'\n origin_ip: ip(\"0.0.0.0\") # default to unspecified ip addr\n attribute_bindings:\n # Fill the new attributes from the adapter produced output.\n # $out refers to an instance of OutputTemplate message\n source.ip: $out.source_pod_ip | ip(\"0.0.0.0\")\n source.labels: $out.source_labels | emptyStringMap()\n source.namespace: $out.source_namespace | \"default\"\n source.service: $out.source_service | \"unknown\"\n source.serviceAccount: $out.source_service_account_name | \"unknown\"\n destination.ip: $out.destination_pod_ip | ip(\"0.0.0.0\")\n destination.labels: $out.destination_labels | emptyStringMap()\n destination.namespace: $out.destination_namespace | \"default\"\n destination.service: $out.destination_service | \"unknown\"\n destination.serviceAccount: $out.destination_service_account_name | \"unknown\"\n ---\n # Configuration needed by Mixer.\n # Mixer cluster is delivered via CDS\n # Specify mixer cluster settings\n apiVersion: networking.istio.io/v1alpha3\n kind: DestinationRule\n metadata:\n name: istio-policy\n namespace: istio-system\n spec:\n host: istio-policy.istio-system.svc.cluster.local\n trafficPolicy:\n connectionPool:\n http:\n http2MaxRequests: 10000\n maxRequestsPerConnection: 10000\n ---\n apiVersion: networking.istio.io/v1alpha3\n kind: DestinationRule\n metadata:\n name: istio-telemetry\n namespace: istio-system\n spec:\n host: istio-telemetry.istio-system.svc.cluster.local\n trafficPolicy:\n connectionPool:\n http:\n http2MaxRequests: 10000\n maxRequestsPerConnection: 10000\n ---\n \n\n---\n# Source: istio/charts/prometheus/templates/configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: prometheus\n namespace: istio-system\n labels:\n app: prometheus\n chart: prometheus-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\ndata:\n prometheus.yml: |-\n global:\n scrape_interval: 15s\n scrape_configs:\n\n - job_name: 'istio-mesh'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n\n kubernetes_sd_configs:\n - role: endpoints\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-system;istio-telemetry;prometheus\n\n - job_name: 'envoy'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-system;istio-statsd-prom-bridge;statsd-prom\n\n - job_name: 'istio-policy'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-system;istio-policy;http-monitoring\n\n - job_name: 'istio-telemetry'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-system;istio-telemetry;http-monitoring\n\n - job_name: 'pilot'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-system;istio-pilot;http-monitoring\n\n # scrape config for API servers\n - job_name: 'kubernetes-apiservers'\n kubernetes_sd_configs:\n - role: endpoints\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n relabel_configs:\n - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: default;kubernetes;https\n\n # scrape config for nodes (kubelet)\n - job_name: 'kubernetes-nodes'\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n kubernetes_sd_configs:\n - role: node\n relabel_configs:\n - action: labelmap\n regex: __meta_kubernetes_node_label_(.+)\n - target_label: __address__\n replacement: kubernetes.default.svc:443\n - source_labels: [__meta_kubernetes_node_name]\n regex: (.+)\n target_label: __metrics_path__\n replacement: /api/v1/nodes/${1}/proxy/metrics\n\n # Scrape config for Kubelet cAdvisor.\n #\n # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics\n # (those whose names begin with 'container_') have been removed from the\n # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to\n # retrieve those metrics.\n #\n # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor\n # HTTP endpoint; use \"replacement: /api/v1/nodes/${1}:4194/proxy/metrics\"\n # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with\n # the --cadvisor-port=0 Kubelet flag).\n #\n # This job is not necessary and should be removed in Kubernetes 1.6 and\n # earlier versions, or it will cause the metrics to be scraped twice.\n - job_name: 'kubernetes-cadvisor'\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n kubernetes_sd_configs:\n - role: node\n relabel_configs:\n - action: labelmap\n regex: __meta_kubernetes_node_label_(.+)\n - target_label: __address__\n replacement: kubernetes.default.svc:443\n - source_labels: [__meta_kubernetes_node_name]\n regex: (.+)\n target_label: __metrics_path__\n replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor\n\n # scrape config for service endpoints.\n - job_name: 'kubernetes-service-endpoints'\n kubernetes_sd_configs:\n - role: endpoints\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]\n action: replace\n target_label: __scheme__\n regex: (https?)\n - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]\n action: replace\n target_label: __metrics_path__\n regex: (.+)\n - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]\n action: replace\n target_label: __address__\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n - action: labelmap\n regex: __meta_kubernetes_service_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: kubernetes_namespace\n - source_labels: [__meta_kubernetes_service_name]\n action: replace\n target_label: kubernetes_name\n\n # Example scrape config for pods\n - job_name: 'kubernetes-pods'\n kubernetes_sd_configs:\n - role: pod\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]\n action: replace\n target_label: __metrics_path__\n regex: (.+)\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]\n action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod_name\n\n---\n# Source: istio/templates/configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio\n namespace: istio-system\n labels:\n app: istio\n chart: istio-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\ndata:\n mesh: |-\n #\n # Edit this list to avoid using mTLS to connect to these services.\n # Typically, these are control services (e.g kubernetes API server) that don't have istio sidecar\n # to transparently terminate mTLS authentication.\n # mtlsExcludedServices: [\"kubernetes.default.svc.cluster.local\"]\n\n # Set the following variable to true to disable policy checks by the Mixer.\n # Note that metrics will still be reported to the Mixer.\n disablePolicyChecks: false\n # Set enableTracing to false to disable request tracing.\n enableTracing: true\n #\n # To disable the mixer completely (including metrics), comment out\n # the following lines\n mixerCheckServer: istio-policy.istio-system.svc.cluster.local:15004\n mixerReportServer: istio-telemetry.istio-system.svc.cluster.local:15004\n # This is the ingress service name, update if you used a different name\n ingressService: istio-ingress\n #\n # Along with discoveryRefreshDelay, this setting determines how\n # frequently should Envoy fetch and update its internal configuration\n # from istio Pilot. Lower refresh delay results in higher CPU\n # utilization and potential performance loss in exchange for faster\n # convergence. Tweak this value according to your setup.\n rdsRefreshDelay: 10s\n #\n defaultConfig:\n # NOTE: If you change any values in this section, make sure to make\n # the same changes in start up args in istio-ingress pods.\n # See rdsRefreshDelay for explanation about this setting.\n discoveryRefreshDelay: 10s\n #\n # TCP connection timeout between Envoy & the application, and between Envoys.\n connectTimeout: 10s\n #\n ### ADVANCED SETTINGS #############\n # Where should envoy's configuration be stored in the istio-proxy container\n configPath: \"/etc/istio/proxy\"\n binaryPath: \"/usr/local/bin/envoy\"\n # The pseudo service name used for Envoy.\n serviceCluster: istio-proxy\n # These settings that determine how long an old Envoy\n # process should be kept alive after an occasional reload.\n drainDuration: 45s\n parentShutdownDuration: 1m0s\n #\n # The mode used to redirect inbound connections to Envoy. This setting\n # has no effect on outbound traffic: iptables REDIRECT is always used for\n # outbound connections.\n # If \"REDIRECT\", use iptables REDIRECT to NAT and redirect to Envoy.\n # The \"REDIRECT\" mode loses source addresses during redirection.\n # If \"TPROXY\", use iptables TPROXY to redirect to Envoy.\n # The \"TPROXY\" mode preserves both the source and destination IP\n # addresses and ports, so that they can be used for advanced filtering\n # and manipulation.\n # The \"TPROXY\" mode also configures the sidecar to run with the\n # CAP_NET_ADMIN capability, which is required to use TPROXY.\n #interceptionMode: REDIRECT\n #\n # Port where Envoy listens (on local host) for admin commands\n # You can exec into the istio-proxy container in a pod and\n # curl the admin port (curl http://localhost:15000/) to obtain\n # diagnostic information from Envoy. See\n # https://lyft.github.io/envoy/docs/operations/admin.html\n # for more details\n proxyAdminPort: 15000\n #\n # Zipkin trace collector\n zipkinAddress: zipkin.istio-system:9411\n #\n # Statsd metrics collector converts statsd metrics into Prometheus metrics.\n statsdUdpAddress: istio-statsd-prom-bridge.istio-system:9125\n #\n # Mutual TLS authentication between sidecars and istio control plane.\n controlPlaneAuthPolicy: NONE\n #\n # Address where istio Pilot service is running\n discoveryAddress: istio-pilot.istio-system:15007\n\n---\n# Source: istio/templates/sidecar-injector-configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio-sidecar-injector\n namespace: istio-system\n labels:\n app: istio\n chart: istio-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: sidecar-injector\ndata:\n config: |-\n policy: enabled\n template: |-\n initContainers:\n - name: istio-init\n image: docker.io/istio/proxy_init:0.8.0\n args:\n - \"-p\"\n - [[ .MeshConfig.ProxyListenPort ]]\n - \"-u\"\n - 1337\n - \"-m\"\n - [[ or (index .ObjectMeta.Annotations \"sidecar.istio.io/interceptionMode\") .ProxyConfig.InterceptionMode.String ]]\n - \"-i\"\n [[ if (isset .ObjectMeta.Annotations \"traffic.sidecar.istio.io/includeOutboundIPRanges\") -]]\n - \"[[ index .ObjectMeta.Annotations \"traffic.sidecar.istio.io/includeOutboundIPRanges\" ]]\"\n [[ else -]]\n - \"*\"\n [[ end -]]\n - \"-x\"\n [[ if (isset .ObjectMeta.Annotations \"traffic.sidecar.istio.io/excludeOutboundIPRanges\") -]]\n - \"[[ index .ObjectMeta.Annotations \"traffic.sidecar.istio.io/excludeOutboundIPRanges\" ]]\"\n [[ else -]]\n - \"\"\n [[ end -]]\n - \"-b\"\n [[ if (isset .ObjectMeta.Annotations \"traffic.sidecar.istio.io/includeInboundPorts\") -]]\n - \"[[ index .ObjectMeta.Annotations \"traffic.sidecar.istio.io/includeInboundPorts\" ]]\"\n [[ else -]]\n - [[ range .Spec.Containers -]][[ range .Ports -]][[ .ContainerPort -]], [[ end -]][[ end -]][[ end]]\n - \"-d\"\n [[ if (isset .ObjectMeta.Annotations \"traffic.sidecar.istio.io/excludeInboundPorts\") -]]\n - \"[[ index .ObjectMeta.Annotations \"traffic.sidecar.istio.io/excludeInboundPorts\" ]]\"\n [[ else -]]\n - \"\"\n [[ end -]]\n imagePullPolicy: IfNotPresent\n securityContext:\n capabilities:\n add:\n - NET_ADMIN\n privileged: true\n restartPolicy: Always\n \n containers:\n - name: istio-proxy\n image: [[ if (isset .ObjectMeta.Annotations \"sidecar.istio.io/proxyImage\") -]]\n \"[[ index .ObjectMeta.Annotations \"sidecar.istio.io/proxyImage\" ]]\"\n [[ else -]]\n docker.io/istio/proxyv2:0.8.0\n [[ end -]]\n args:\n - proxy\n - sidecar\n - --configPath\n - [[ .ProxyConfig.ConfigPath ]]\n - --binaryPath\n - [[ .ProxyConfig.BinaryPath ]]\n - --serviceCluster\n [[ if ne \"\" (index .ObjectMeta.Labels \"app\") -]]\n - [[ index .ObjectMeta.Labels \"app\" ]]\n [[ else -]]\n - \"istio-proxy\"\n [[ end -]]\n - --drainDuration\n - [[ formatDuration .ProxyConfig.DrainDuration ]]\n - --parentShutdownDuration\n - [[ formatDuration .ProxyConfig.ParentShutdownDuration ]]\n - --discoveryAddress\n - [[ .ProxyConfig.DiscoveryAddress ]]\n - --discoveryRefreshDelay\n - [[ formatDuration .ProxyConfig.DiscoveryRefreshDelay ]]\n - --zipkinAddress\n - [[ .ProxyConfig.ZipkinAddress ]]\n - --connectTimeout\n - [[ formatDuration .ProxyConfig.ConnectTimeout ]]\n - --statsdUdpAddress\n - [[ .ProxyConfig.StatsdUdpAddress ]]\n - --proxyAdminPort\n - [[ .ProxyConfig.ProxyAdminPort ]]\n - --controlPlaneAuthPolicy\n - [[ .ProxyConfig.ControlPlaneAuthPolicy ]]\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: ISTIO_META_POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: ISTIO_META_INTERCEPTION_MODE\n value: [[ or (index .ObjectMeta.Annotations \"sidecar.istio.io/interceptionMode\") .ProxyConfig.InterceptionMode.String ]]\n imagePullPolicy: IfNotPresent\n securityContext:\n privileged: false\n readOnlyRootFilesystem: true\n [[ if eq (or (index .ObjectMeta.Annotations \"sidecar.istio.io/interceptionMode\") .ProxyConfig.InterceptionMode.String) \"TPROXY\" -]]\n capabilities:\n add:\n - NET_ADMIN\n [[ else -]]\n runAsUser: 1337\n [[ end -]]\n restartPolicy: Always\n resources:\n requests:\n cpu: 100m\n memory: 128Mi\n \n volumeMounts:\n - mountPath: /etc/istio/proxy\n name: istio-envoy\n - mountPath: /etc/certs/\n name: istio-certs\n readOnly: true\n volumes:\n - emptyDir:\n medium: Memory\n name: istio-envoy\n - name: istio-certs\n secret:\n optional: true\n [[ if eq .Spec.ServiceAccountName \"\" -]]\n secretName: istio.default\n [[ else -]]\n secretName: [[ printf \"istio.%s\" .Spec.ServiceAccountName ]]\n [[ end -]]\n\n\n---\n# Source: istio/charts/egressgateway/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-egressgateway-service-account\n namespace: istio-system\n labels:\n app: egressgateway\n chart: egressgateway-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\n\n---\n# Source: istio/charts/ingressgateway/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-ingressgateway-service-account\n namespace: istio-system\n labels:\n app: ingressgateway\n chart: ingressgateway-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\n\n---\n# Source: istio/charts/mixer/templates/create-custom-resources-job.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-mixer-post-install-account\n namespace: istio-system\n labels:\n app: mixer\n chart: mixer-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-mixer-post-install-istio-system\n namespace: istio-system\n labels:\n app: mixer\n chart: mixer-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nrules:\n- apiGroups: [\"config.istio.io\"] # istio CRD watcher\n resources: [\"*\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"patch\"]\n- apiGroups: [\"networking.istio.io\"] # needed to create mixer destination rules\n resources: [\"*\"]\n verbs: [\"*\"]\n- apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources: [\"configmaps\", \"endpoints\", \"pods\", \"services\", \"namespaces\", \"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-mixer-post-install-role-binding-istio-system\n labels:\n app: mixer\n chart: mixer-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-mixer-post-install-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-mixer-post-install-account\n namespace: istio-system\n---\n\napiVersion: batch/v1\nkind: Job\nmetadata:\n name: istio-mixer-post-install\n namespace: istio-system\n annotations:\n \"helm.sh/hook\": post-install\n \"helm.sh/hook-delete-policy\": before-hook-creation\n labels:\n app: mixer\n chart: mixer-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n template:\n metadata:\n name: istio-mixer-post-install\n labels:\n app: mixer\n release: RELEASE-NAME\n spec:\n serviceAccountName: istio-mixer-post-install-account\n containers:\n - name: hyperkube\n image: \"quay.io/coreos/hyperkube:v1.7.6_coreos.0\"\n command:\n - ./kubectl\n - apply\n - -f\n - /tmp/mixer/custom-resources.yaml\n volumeMounts:\n - mountPath: \"/tmp/mixer\"\n name: tmp-configmap-mixer\n volumes:\n - name: tmp-configmap-mixer\n configMap:\n name: istio-mixer-custom-resources\n restartPolicy: Never # CRD might take some time till they are available to consume\n\n---\n# Source: istio/charts/mixer/templates/serviceaccount.yaml\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-mixer-service-account\n namespace: istio-system\n labels:\n app: mixer\n chart: mixer-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\n\n---\n# Source: istio/charts/pilot/templates/serviceaccount.yaml\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-pilot-service-account\n namespace: istio-system\n labels:\n app: istio-pilot\n chart: pilot-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\n\n---\n# Source: istio/charts/prometheus/templates/serviceaccount.yaml\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: prometheus\n namespace: istio-system\n\n---\n# Source: istio/charts/security/templates/serviceaccount.yaml\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-citadel-service-account\n namespace: istio-system\n labels:\n app: security\n chart: security-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-cleanup-old-ca-service-account\n namespace: istio-system\n labels:\n app: security\n chart: security-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-sidecar-injector-service-account\n namespace: istio-system\n labels:\n app: istio-sidecar-injector\n chart: sidecarInjectorWebhook-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\n\n---\n# Source: istio/charts/mixer/templates/crds.yaml\n# Mixer CRDs\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: rules.config.istio.io\n labels:\n app: mixer\n package: istio.io.mixer\n istio: core\nspec:\n group: config.istio.io\n names:\n kind: rule\n plural: rules\n singular: rule\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: attributemanifests.config.istio.io\n labels:\n app: mixer\n package: istio.io.mixer\n istio: core\nspec:\n group: config.istio.io\n names:\n kind: attributemanifest\n plural: attributemanifests\n singular: attributemanifest\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: circonuses.config.istio.io\n labels:\n app: mixer\n package: circonus\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: circonus\n plural: circonuses\n singular: circonus\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: deniers.config.istio.io\n labels:\n app: mixer\n package: denier\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: denier\n plural: deniers\n singular: denier\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: fluentds.config.istio.io\n labels:\n app: mixer\n package: fluentd\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: fluentd\n plural: fluentds\n singular: fluentd\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: kubernetesenvs.config.istio.io\n labels:\n app: mixer\n package: kubernetesenv\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: kubernetesenv\n plural: kubernetesenvs\n singular: kubernetesenv\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: listcheckers.config.istio.io\n labels:\n app: mixer\n package: listchecker\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: listchecker\n plural: listcheckers\n singular: listchecker\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: memquotas.config.istio.io\n labels:\n app: mixer\n package: memquota\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: memquota\n plural: memquotas\n singular: memquota\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: noops.config.istio.io\n labels:\n app: mixer\n package: noop\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: noop\n plural: noops\n singular: noop\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: opas.config.istio.io\n labels:\n app: mixer\n package: opa\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: opa\n plural: opas\n singular: opa\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: prometheuses.config.istio.io\n labels:\n app: mixer\n package: prometheus\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: prometheus\n plural: prometheuses\n singular: prometheus\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: rbacs.config.istio.io\n labels:\n app: mixer\n package: rbac\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: rbac\n plural: rbacs\n singular: rbac\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: servicecontrols.config.istio.io\n labels:\n app: mixer\n package: servicecontrol\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: servicecontrol\n plural: servicecontrols\n singular: servicecontrol\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: solarwindses.config.istio.io\n labels:\n app: mixer\n package: solarwinds\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: solarwinds\n plural: solarwindses\n singular: solarwinds\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: stackdrivers.config.istio.io\n labels:\n app: mixer\n package: stackdriver\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: stackdriver\n plural: stackdrivers\n singular: stackdriver\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: statsds.config.istio.io\n labels:\n app: mixer\n package: statsd\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: statsd\n plural: statsds\n singular: statsd\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: stdios.config.istio.io\n labels:\n app: mixer\n package: stdio\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: stdio\n plural: stdios\n singular: stdio\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: apikeys.config.istio.io\n labels:\n app: mixer\n package: apikey\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: apikey\n plural: apikeys\n singular: apikey\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: authorizations.config.istio.io\n labels:\n app: mixer\n package: authorization\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: authorization\n plural: authorizations\n singular: authorization\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: checknothings.config.istio.io\n labels:\n app: mixer\n package: checknothing\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: checknothing\n plural: checknothings\n singular: checknothing\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: kuberneteses.config.istio.io\n labels:\n app: mixer\n package: adapter.template.kubernetes\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: kubernetes\n plural: kuberneteses\n singular: kubernetes\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: listentries.config.istio.io\n labels:\n app: mixer\n package: listentry\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: listentry\n plural: listentries\n singular: listentry\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: logentries.config.istio.io\n labels:\n app: mixer\n package: logentry\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: logentry\n plural: logentries\n singular: logentry\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: metrics.config.istio.io\n labels:\n app: mixer\n package: metric\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: metric\n plural: metrics\n singular: metric\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: quotas.config.istio.io\n labels:\n app: mixer\n package: quota\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: quota\n plural: quotas\n singular: quota\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: reportnothings.config.istio.io\n labels:\n app: mixer\n package: reportnothing\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: reportnothing\n plural: reportnothings\n singular: reportnothing\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: servicecontrolreports.config.istio.io\n labels:\n app: mixer\n package: servicecontrolreport\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: servicecontrolreport\n plural: servicecontrolreports\n singular: servicecontrolreport\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: tracespans.config.istio.io\n labels:\n app: mixer\n package: tracespan\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: tracespan\n plural: tracespans\n singular: tracespan\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: serviceroles.config.istio.io\n labels:\n app: mixer\n package: istio.io.mixer\n istio: rbac\nspec:\n group: config.istio.io\n names:\n kind: ServiceRole\n plural: serviceroles\n singular: servicerole\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: servicerolebindings.config.istio.io\n labels:\n app: mixer\n package: istio.io.mixer\n istio: rbac\nspec:\n group: config.istio.io\n names:\n kind: ServiceRoleBinding\n plural: servicerolebindings\n singular: servicerolebinding\n scope: Namespaced\n version: v1alpha2\n\n---\n# Source: istio/charts/pilot/templates/crds.yaml\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: destinationpolicies.config.istio.io\n labels:\n app: istio-pilot\nspec:\n group: config.istio.io\n names:\n kind: DestinationPolicy\n listKind: DestinationPolicyList\n plural: destinationpolicies\n singular: destinationpolicy\n scope: Namespaced\n version: v1alpha2\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: egressrules.config.istio.io\n labels:\n app: istio-pilot\nspec:\n group: config.istio.io\n names:\n kind: EgressRule\n listKind: EgressRuleList\n plural: egressrules\n singular: egressrule\n scope: Namespaced\n version: v1alpha2\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: routerules.config.istio.io\n labels:\n app: istio-pilot\nspec:\n group: config.istio.io\n names:\n kind: RouteRule\n listKind: RouteRuleList\n plural: routerules\n singular: routerule\n scope: Namespaced\n version: v1alpha2\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: virtualservices.networking.istio.io\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: VirtualService\n listKind: VirtualServiceList\n plural: virtualservices\n singular: virtualservice\n scope: Namespaced\n version: v1alpha3\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: destinationrules.networking.istio.io\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: DestinationRule\n listKind: DestinationRuleList\n plural: destinationrules\n singular: destinationrule\n scope: Namespaced\n version: v1alpha3\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: serviceentries.networking.istio.io\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: ServiceEntry\n listKind: ServiceEntryList\n plural: serviceentries\n singular: serviceentry\n scope: Namespaced\n version: v1alpha3\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: gateways.networking.istio.io\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: Gateway\n plural: gateways\n singular: gateway\n scope: Namespaced\n version: v1alpha3\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: policies.authentication.istio.io\nspec:\n group: authentication.istio.io\n names:\n kind: Policy\n plural: policies\n singular: policy\n scope: Namespaced\n version: v1alpha1\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: httpapispecbindings.config.istio.io\nspec:\n group: config.istio.io\n names:\n kind: HTTPAPISpecBinding\n plural: httpapispecbindings\n singular: httpapispecbinding\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: httpapispecs.config.istio.io\nspec:\n group: config.istio.io\n names:\n kind: HTTPAPISpec\n plural: httpapispecs\n singular: httpapispec\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: quotaspecbindings.config.istio.io\nspec:\n group: config.istio.io\n names:\n kind: QuotaSpecBinding\n plural: quotaspecbindings\n singular: quotaspecbinding\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: quotaspecs.config.istio.io\nspec:\n group: config.istio.io\n names:\n kind: QuotaSpec\n plural: quotaspecs\n singular: quotaspec\n scope: Namespaced\n version: v1alpha2\n\n\n---\n# Source: istio/charts/mixer/templates/clusterrole.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-mixer-istio-system\n namespace: istio-system\n labels:\n app: mixer\n chart: mixer-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nrules:\n- apiGroups: [\"config.istio.io\"] # istio CRD watcher\n resources: [\"*\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"patch\"]\n- apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources: [\"configmaps\", \"endpoints\", \"pods\", \"services\", \"namespaces\", \"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n\n---\n# Source: istio/charts/pilot/templates/clusterrole.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-pilot-istio-system\n namespace: istio-system\n labels:\n app: istio-pilot\n chart: pilot-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nrules:\n- apiGroups: [\"config.istio.io\"]\n resources: [\"*\"]\n verbs: [\"*\"]\n- apiGroups: [\"networking.istio.io\"]\n resources: [\"*\"]\n verbs: [\"*\"]\n- apiGroups: [\"authentication.istio.io\"]\n resources: [\"*\"]\n verbs: [\"*\"]\n- apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"*\"]\n- apiGroups: [\"extensions\"]\n resources: [\"thirdpartyresources\", \"thirdpartyresources.extensions\", \"ingresses\", \"ingresses/status\"]\n verbs: [\"*\"]\n- apiGroups: [\"\"]\n resources: [\"configmaps\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\"]\n- apiGroups: [\"\"]\n resources: [\"endpoints\", \"pods\", \"services\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources: [\"namespaces\", \"nodes\", \"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n\n---\n# Source: istio/charts/prometheus/templates/clusterrole.yaml\n\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: prometheus-istio-system\n namespace: istio-system\nrules:\n- apiGroups: [\"\"]\n resources:\n - nodes\n - services\n - endpoints\n - pods\n - nodes/proxy\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources:\n - configmaps\n verbs: [\"get\"]\n- nonResourceURLs: [\"/metrics\"]\n verbs: [\"get\"]\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: prometheus-istio-system\n namespace: istio-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: prometheus-istio-system\nsubjects:\n- kind: ServiceAccount\n name: prometheus\n namespace: istio-system\n---\n\n\n---\n# Source: istio/charts/security/templates/clusterrole.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-citadel-istio-system\n namespace: istio-system\n labels:\n app: security\n chart: security-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nrules:\n- apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"create\", \"get\", \"watch\", \"list\", \"update\", \"delete\"]\n- apiGroups: [\"\"]\n resources: [\"serviceaccounts\"]\n verbs: [\"get\", \"watch\", \"list\"]\n- apiGroups: [\"\"]\n resources: [\"services\"]\n verbs: [\"get\", \"watch\", \"list\"]\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: Role\nmetadata:\n name: istio-cleanup-old-ca-istio-system\n namespace: istio-system\n labels:\n app: security\n chart: security-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nrules:\n- apiGroups: [\"\"]\n resources: [\"deployments\", \"serviceaccounts\", \"services\"]\n verbs: [\"get\", \"delete\"]\n- apiGroups: [\"extensions\"]\n resources: [\"deployments\", \"replicasets\"]\n verbs: [\"get\", \"list\", \"update\", \"delete\"]\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/clusterrole.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-sidecar-injector-istio-system\n labels:\n app: istio-sidecar-injector\n chart: sidecarInjectorWebhook-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nrules:\n- apiGroups: [\"*\"]\n resources: [\"configmaps\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"admissionregistration.k8s.io\"]\n resources: [\"mutatingwebhookconfigurations\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n\n---\n# Source: istio/charts/mixer/templates/clusterrolebinding.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-mixer-admin-role-binding-istio-system\n labels:\n app: mixer\n chart: mixer-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-mixer-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-mixer-service-account\n namespace: istio-system\n\n---\n# Source: istio/charts/pilot/templates/clusterrolebinding.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-pilot-istio-system\n labels:\n app: istio-pilot\n chart: pilot-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-pilot-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-pilot-service-account\n namespace: istio-system\n\n---\n# Source: istio/charts/security/templates/clusterrolebinding.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-citadel-istio-system\n labels:\n app: security\n chart: security-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-citadel-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-citadel-service-account\n namespace: istio-system\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: RoleBinding\nmetadata:\n name: istio-cleanup-old-ca-istio-system\n namespace: istio-system\n labels:\n app: security\n chart: security-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: istio-cleanup-old-ca-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-cleanup-old-ca-service-account\n namespace: istio-system\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/clusterrolebinding.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-sidecar-injector-admin-role-binding-istio-system\n labels:\n app: istio-sidecar-injector\n chart: sidecarInjectorWebhook-0.8.0\n heritage: Tiller\n release: RELEASE-NAME\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-sidecar-injector-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-sidecar-injector-service-account\n namespace: istio-system\n---\n# Source: istio/charts/egressgateway/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-egressgateway\n namespace: istio-system \n labels:\n chart: egressgateway-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: egressgateway\nspec:\n type: ClusterIP\n selector:\n istio: egressgateway\n ports:\n -\n name: http\n port: 80\n -\n name: https\n port: 443\n\n---\n# Source: istio/charts/grafana/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: grafana\n namespace: istio-system\n annotations:\n auth.istio.io/3000: NONE\n labels:\n app: grafana\n chart: grafana-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n type: ClusterIP\n ports:\n - port: 3000\n targetPort: 3000\n protocol: TCP\n name: http\n selector:\n app: grafana\n\n---\n# Source: istio/charts/ingressgateway/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-ingressgateway\n namespace: istio-system \n labels:\n chart: ingressgateway-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: ingressgateway\nspec:\n type: NodePort\n selector:\n istio: ingressgateway\n ports:\n -\n name: http\n nodePort: 31380\n port: 80\n -\n name: https\n nodePort: 31390\n port: 443\n -\n name: tcp\n nodePort: 31400\n port: 31400\n\n---\n# Source: istio/charts/mixer/templates/service.yaml\n\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-policy\n namespace: istio-system\n labels:\n chart: mixer-0.8.0\n release: RELEASE-NAME\n istio: mixer\nspec:\n ports:\n - name: grpc-mixer\n port: 9091\n - name: grpc-mixer-mtls\n port: 15004\n - name: http-monitoring\n port: 9093\n selector:\n istio: mixer\n istio-mixer-type: policy\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-telemetry\n namespace: istio-system\n labels:\n chart: mixer-0.8.0\n release: RELEASE-NAME\n istio: mixer\nspec:\n ports:\n - name: grpc-mixer\n port: 9091\n - name: grpc-mixer-mtls\n port: 15004\n - name: http-monitoring\n port: 9093\n - name: prometheus\n port: 42422\n selector:\n istio: mixer\n istio-mixer-type: telemetry\n---\n\n---\n# Source: istio/charts/mixer/templates/statsdtoprom.yaml\n\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-statsd-prom-bridge\n namespace: istio-system\n labels:\n chart: mixer-0.8.0\n release: RELEASE-NAME\n istio: statsd-prom-bridge\nspec:\n ports:\n - name: statsd-prom\n port: 9102\n - name: statsd-udp\n port: 9125\n protocol: UDP\n selector:\n istio: statsd-prom-bridge\n\n---\n\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-statsd-prom-bridge\n namespace: istio-system\n labels:\n chart: mixer-0.8.0\n release: RELEASE-NAME\n istio: mixer\nspec:\n template:\n metadata:\n labels:\n istio: statsd-prom-bridge\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: istio-mixer-service-account\n volumes:\n - name: config-volume\n configMap:\n name: istio-statsd-prom-bridge\n containers:\n - name: statsd-prom-bridge\n image: \"prom/statsd-exporter:latest\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9102\n - containerPort: 9125\n protocol: UDP\n args:\n - '-statsd.mapping-config=/etc/statsd/mapping.conf'\n resources:\n {}\n \n volumeMounts:\n - name: config-volume\n mountPath: /etc/statsd\n\n---\n# Source: istio/charts/pilot/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-pilot\n namespace: istio-system\n labels:\n app: istio-pilot\n chart: pilot-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n ports:\n - port: 15003\n name: http-old-discovery # mTLS or non-mTLS depending on auth setting\n - port: 15005\n name: https-discovery # always mTLS\n - port: 15007\n name: http-discovery # always plain-text\n - port: 15010\n name: grpc-xds # direct\n - port: 15011\n name: https-xds # mTLS\n - port: 8080\n name: http-legacy-discovery # direct\n - port: 9093\n name: http-monitoring\n selector:\n istio: pilot\n\n---\n# Source: istio/charts/prometheus/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: prometheus\n namespace: istio-system\n annotations:\n prometheus.io/scrape: 'true'\n labels:\n name: prometheus\nspec:\n selector:\n app: prometheus\n ports:\n - name: http-prometheus\n protocol: TCP\n port: 9090\n\n---\n# Source: istio/charts/security/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n # we use the normal name here (e.g. 'prometheus')\n # as grafana is configured to use this as a data source\n name: istio-citadel\n namespace: istio-system\n labels:\n app: istio-citadel\nspec:\n ports:\n - name: grpc-citadel\n port: 8060\n targetPort: 8060\n protocol: TCP\n - name: http-monitoring\n port: 9093\n selector:\n istio: citadel\n\n---\n# Source: istio/charts/servicegraph/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: servicegraph\n namespace: istio-system\n labels:\n app: servicegraph\n chart: servicegraph-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n type: ClusterIP\n ports:\n - port: 8088\n targetPort: 8088\n protocol: TCP\n name: http\n selector:\n app: servicegraph\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-sidecar-injector\n namespace: istio-system\n labels:\n istio: sidecar-injector\nspec:\n ports:\n - port: 443\n selector:\n istio: sidecar-injector\n\n---\n# Source: istio/charts/egressgateway/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-egressgateway\n namespace: istio-system\n labels:\n app: egressgateway\n chart: egressgateway-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: egressgateway\nspec:\n replicas: \n template:\n metadata:\n labels:\n istio: egressgateway\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: istio-egressgateway-service-account\n containers:\n - name: egressgateway\n image: \"docker.io/istio/proxyv2:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 80\n - containerPort: 443\n args:\n - proxy\n - router\n - -v\n - \"2\"\n - --discoveryRefreshDelay\n - '1s' #discoveryRefreshDelay\n - --drainDuration\n - '45s' #drainDuration\n - --parentShutdownDuration\n - '1m0s' #parentShutdownDuration\n - --connectTimeout\n - '10s' #connectTimeout\n - --serviceCluster\n - istio-egressgateway\n - --zipkinAddress\n - zipkin:9411\n - --statsdUdpAddress\n - istio-statsd-prom-bridge:9125\n - --proxyAdminPort\n - \"15000\"\n - --controlPlaneAuthPolicy\n - NONE\n - --discoveryAddress\n - istio-pilot:8080\n resources:\n {}\n \n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: ISTIO_META_POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n volumes:\n - name: istio-certs\n secret:\n secretName: \"istio.default\"\n optional: true\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/grafana/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: grafana\n namespace: istio-system\n labels:\n app: grafana\n chart: grafana-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n app: grafana\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n containers:\n - name: grafana\n image: \"docker.io/istio/grafana:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 3000\n readinessProbe:\n httpGet:\n path: /login\n port: 3000\n env:\n - name: GRAFANA_PORT\n value: \"3000\"\n - name: GF_AUTH_BASIC_ENABLED\n value: \"false\"\n - name: GF_AUTH_ANONYMOUS_ENABLED\n value: \"true\"\n - name: GF_AUTH_ANONYMOUS_ORG_ROLE\n value: Admin\n - name: GF_PATHS_DATA\n value: /data/grafana\n resources:\n {}\n \n volumeMounts:\n - name: data\n mountPath: /data/grafana\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n volumes:\n - name: data\n emptyDir: {}\n---\n# Source: istio/charts/ingressgateway/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-ingressgateway\n namespace: istio-system\n labels:\n app: ingressgateway\n chart: ingressgateway-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: ingressgateway\nspec:\n replicas: \n template:\n metadata:\n labels:\n istio: ingressgateway\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: istio-ingressgateway-service-account\n containers:\n - name: ingressgateway\n image: \"docker.io/istio/proxyv2:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 80\n - containerPort: 443\n - containerPort: 31400\n args:\n - proxy\n - router\n - -v\n - \"2\"\n - --discoveryRefreshDelay\n - '1s' #discoveryRefreshDelay\n - --drainDuration\n - '45s' #drainDuration\n - --parentShutdownDuration\n - '1m0s' #parentShutdownDuration\n - --connectTimeout\n - '10s' #connectTimeout\n - --serviceCluster\n - istio-ingressgateway\n - --zipkinAddress\n - zipkin:9411\n - --statsdUdpAddress\n - istio-statsd-prom-bridge:9125\n - --proxyAdminPort\n - \"15000\"\n - --controlPlaneAuthPolicy\n - NONE\n - --discoveryAddress\n - istio-pilot:8080\n resources:\n {}\n \n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n - name: ISTIO_META_POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n - name: ingressgateway-certs\n mountPath: \"/etc/istio/ingressgateway-certs\"\n readOnly: true\n volumes:\n - name: istio-certs\n secret:\n secretName: \"istio.default\"\n optional: true\n - name: ingressgateway-certs\n secret:\n secretName: \"istio-ingressgateway-certs\"\n optional: true\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/mixer/templates/deployment.yaml\n\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-policy\n namespace: istio-system\n labels:\n chart: mixer-0.8.0\n release: RELEASE-NAME\n istio: mixer\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n istio: mixer\n istio-mixer-type: policy\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: istio-mixer-service-account\n volumes:\n - name: istio-certs\n secret:\n secretName: istio.istio-mixer-service-account\n optional: true\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n containers:\n - name: mixer\n image: \"docker.io/istio/mixer:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9092\n - containerPort: 9093\n - containerPort: 42422\n args:\n - --address\n - tcp://127.0.0.1:9092\n - --configStoreURL=k8s://\n - --configDefaultNamespace=istio-system\n - --trace_zipkin_url=http://zipkin:9411/api/v1/spans\n resources:\n {}\n \n - name: istio-proxy\n image: \"docker.io/istio/proxyv2:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9091\n - containerPort: 15004\n args:\n - proxy\n - --serviceCluster\n - istio-policy\n - --templateFile\n - /etc/istio/proxy/envoy_policy.yaml.tmpl\n - --controlPlaneAuthPolicy\n - NONE\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n resources:\n requests:\n cpu: 100m\n memory: 128Mi\n \n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n\n---\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-telemetry\n namespace: istio-system\n labels:\n chart: mixer-0.8.0\n release: RELEASE-NAME\n istio: mixer\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n istio: mixer\n istio-mixer-type: telemetry\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: istio-mixer-service-account\n volumes:\n - name: istio-certs\n secret:\n secretName: istio.istio-mixer-service-account\n optional: true\n containers:\n - name: mixer\n image: \"docker.io/istio/mixer:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9092\n - containerPort: 9093\n - containerPort: 42422\n args:\n - --address\n - tcp://127.0.0.1:9092\n - --configStoreURL=k8s://\n - --configDefaultNamespace=istio-system\n - --trace_zipkin_url=http://zipkin:9411/api/v1/spans\n resources:\n {}\n \n - name: istio-proxy\n image: \"docker.io/istio/proxyv2:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9091\n - containerPort: 15004\n args:\n - proxy\n - --serviceCluster\n - istio-telemetry\n - --templateFile\n - /etc/istio/proxy/envoy_telemetry.yaml.tmpl\n - --controlPlaneAuthPolicy\n - NONE\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n resources:\n requests:\n cpu: 100m\n memory: 128Mi\n \n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n\n--- \n\n---\n# Source: istio/charts/pilot/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-pilot\n namespace: istio-system\n # TODO: default tempate doesn't have this, which one is right ?\n labels:\n app: istio-pilot\n chart: pilot-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: pilot\n annotations:\n checksum/config-volume: f8da08b6b8c170dde721efd680270b2901e750d4aa186ebb6c22bef5b78a43f9\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n istio: pilot\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: istio-pilot-service-account\n containers:\n - name: discovery\n image: \"docker.io/istio/pilot:0.8.0\"\n imagePullPolicy: IfNotPresent\n args:\n - \"discovery\"\n# TODO(sdake) remove when secrets are automagically registered\n ports:\n - containerPort: 8080\n - containerPort: 15010\n readinessProbe:\n httpGet:\n path: /v1/registration\n port: 8080\n initialDelaySeconds: 30\n periodSeconds: 30\n timeoutSeconds: 5\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: PILOT_THROTTLE\n value: \"500\"\n - name: PILOT_CACHE_SQUASH\n value: \"5\"\n resources:\n {}\n \n volumeMounts:\n - name: config-volume\n mountPath: /etc/istio/config\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n - name: istio-proxy\n image: \"docker.io/istio/proxyv2:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 15003\n - containerPort: 15005\n - containerPort: 15007\n - containerPort: 15011\n args:\n - proxy\n - --serviceCluster\n - istio-pilot\n - --templateFile\n - /etc/istio/proxy/envoy_pilot.yaml.tmpl\n - --controlPlaneAuthPolicy\n - NONE\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n resources:\n requests:\n cpu: 100m\n memory: 128Mi\n \n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n volumes:\n - name: config-volume\n configMap:\n name: istio\n - name: istio-certs\n secret:\n secretName: \"istio.istio-pilot-service-account\"\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/prometheus/templates/deployment.yaml\n# TODO: the original template has service account, roles, etc\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: prometheus\n namespace: istio-system\n labels:\n app: prometheus\n chart: prometheus-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: prometheus\n template:\n metadata:\n labels:\n app: prometheus\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: prometheus\n\n containers:\n - name: prometheus\n image: \"docker.io/prom/prometheus:latest\"\n imagePullPolicy: IfNotPresent\n args:\n - '--storage.tsdb.retention=6h'\n - '--config.file=/etc/prometheus/prometheus.yml'\n ports:\n - containerPort: 9090\n name: http\n livenessProbe:\n httpGet:\n path: /-/healthy\n port: 9090\n readinessProbe:\n httpGet:\n path: /-/ready\n port: 9090\n resources:\n {}\n \n volumeMounts:\n - name: config-volume\n mountPath: /etc/prometheus\n volumes:\n - name: config-volume\n configMap:\n name: prometheus\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/security/templates/deployment.yaml\n# istio CA watching all namespaces\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-citadel\n namespace: istio-system\n labels:\n app: security\n chart: security-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: citadel\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n istio: citadel\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: istio-citadel-service-account\n containers:\n - name: citadel\n image: \"docker.io/istio/citadel:0.8.0\"\n imagePullPolicy: IfNotPresent\n args:\n - --append-dns-names=true\n - --grpc-port=8060\n - --grpc-hostname=citadel\n - --self-signed-ca=true\n - --citadel-storage-namespace=istio-system\n resources:\n {}\n \n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/servicegraph/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: servicegraph\n namespace: istio-system\n labels:\n app: servicegraph\n chart: servicegraph-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n app: servicegraph\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n containers:\n - name: servicegraph\n image: \"docker.io/istio/servicegraph:0.8.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 8088\n args:\n - --prometheusAddr=http://prometheus:9090\n livenessProbe:\n httpGet:\n path: /graph\n port: 8088\n readinessProbe:\n httpGet:\n path: /graph\n port: 8088\n resources:\n {}\n \n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-sidecar-injector\n namespace: istio-system\n labels:\n app: sidecarInjectorWebhook\n chart: sidecarInjectorWebhook-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\n istio: sidecar-injector\nspec:\n replicas: \n template:\n metadata:\n labels:\n istio: sidecar-injector\n spec:\n serviceAccountName: istio-sidecar-injector-service-account\n containers:\n - name: sidecar-injector-webhook\n image: \"docker.io/istio/sidecar_injector:0.8.0\"\n imagePullPolicy: IfNotPresent\n args:\n - --caCertFile=/etc/istio/certs/root-cert.pem\n - --tlsCertFile=/etc/istio/certs/cert-chain.pem\n - --tlsKeyFile=/etc/istio/certs/key.pem\n - --injectConfig=/etc/istio/inject/config\n - --meshConfig=/etc/istio/config/mesh\n - --healthCheckInterval=2s\n - --healthCheckFile=/health\n volumeMounts:\n - name: config-volume\n mountPath: /etc/istio/config\n readOnly: true\n - name: certs\n mountPath: /etc/istio/certs\n readOnly: true\n - name: inject-config\n mountPath: /etc/istio/inject\n readOnly: true\n livenessProbe:\n exec:\n command:\n - /usr/local/bin/sidecar-injector\n - probe\n - --probe-path=/health\n - --interval=2s\n initialDelaySeconds: 4\n periodSeconds: 4\n readinessProbe:\n exec:\n command:\n - /usr/local/bin/sidecar-injector\n - probe\n - --probe-path=/health\n - --interval=2s\n initialDelaySeconds: 4\n periodSeconds: 4\n volumes:\n - name: config-volume\n configMap:\n name: istio\n - name: certs\n secret:\n secretName: istio.istio-sidecar-injector-service-account\n - name: inject-config\n configMap:\n name: istio-sidecar-injector\n items:\n - key: config\n path: config\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/tracing/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-tracing\n namespace: istio-system\n labels:\n app: istio-tracing\n chart: tracing-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n app: jaeger\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n containers:\n - name: jaeger\n image: \"jaegertracing/all-in-one:1.5\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9411\n - containerPort: 16686\n - containerPort: 5775\n protocol: UDP\n - containerPort: 6831\n protocol: UDP\n - containerPort: 6832\n protocol: UDP\n env:\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: COLLECTOR_ZIPKIN_HTTP_PORT\n value: \"9411\"\n - name: MEMORY_MAX_TRACES\n value: \"50000\"\n livenessProbe:\n httpGet:\n path: /\n port: 16686\n readinessProbe:\n httpGet:\n path: /\n port: 16686\n resources:\n {}\n \n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/security/templates/cleanup-old-ca.yaml\n\napiVersion: batch/v1\nkind: Job\nmetadata:\n name: istio-cleanup-old-ca\n namespace: istio-system\n annotations:\n \"helm.sh/hook\": post-install\n \"helm.sh/hook-delete-policy\": hook-succeeded\n labels:\n app: security\n chart: security-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\nspec:\n template:\n metadata:\n name: istio-cleanup-old-ca\n labels:\n app: security\n release: RELEASE-NAME\n spec:\n serviceAccountName: istio-cleanup-old-ca-service-account\n containers:\n - name: hyperkube\n image: \"quay.io/coreos/hyperkube:v1.7.6_coreos.0\"\n command:\n - /bin/bash\n - -c\n - >\n NS=\"-n istio-system\";\n ./kubectl get deploy istio-ca $NS;\n if [[ $? = 0 ]]; then ./kubectl delete deploy istio-ca $NS; fi;\n ./kubectl get serviceaccount istio-ca-service-account $NS;\n if [[ $? = 0 ]]; then ./kubectl delete serviceaccount istio-ca-service-account $NS; fi;\n ./kubectl get service istio-ca-ilb $NS;\n if [[ $? = 0 ]]; then ./kubectl delete service istio-ca-ilb $NS; fi\n restartPolicy: Never\n---\n# Source: istio/charts/egressgateway/templates/autoscale.yaml\n\napiVersion: autoscaling/v2beta1\nkind: HorizontalPodAutoscaler\nmetadata:\n name: istio-egressgateway\n namespace: istio-system\nspec:\n maxReplicas: 1\n minReplicas: 1\n scaleTargetRef:\n apiVersion: apps/v1beta1\n kind: Deployment\n name: istio-egressgateway\n metrics:\n - type: Resource\n resource:\n name: cpu\n targetAverageUtilization: 80\n\n\n---\n# Source: istio/charts/ingressgateway/templates/autoscale.yaml\n\napiVersion: autoscaling/v2beta1\nkind: HorizontalPodAutoscaler\nmetadata:\n name: istio-ingressgateway\n namespace: istio-system\nspec:\n maxReplicas: 1\n minReplicas: 1\n scaleTargetRef:\n apiVersion: apps/v1beta1\n kind: Deployment\n name: istio-ingressgateway\n metrics:\n - type: Resource\n resource:\n name: cpu\n targetAverageUtilization: 80\n\n\n---\n# Source: istio/charts/tracing/templates/service.yaml\napiVersion: v1\nkind: List\nitems:\n- apiVersion: v1\n kind: Service\n metadata:\n name: zipkin\n namespace: istio-system\n labels:\n app: jaeger\n chart: tracing-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\n spec:\n type: ClusterIP\n ports:\n - port: 9411\n targetPort: 9411\n protocol: TCP\n name: http\n selector:\n app: jaeger\n- apiVersion: v1\n kind: Service\n metadata:\n name: tracing\n namespace: istio-system\n labels:\n app: jaeger\n chart: tracing-0.1.0\n release: RELEASE-NAME\n heritage: Tiller\n spec:\n ports:\n - name: query-http\n port: 80\n protocol: TCP\n targetPort: 16686\n selector:\n app: jaeger\n type: LoadBalancer\n\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/mutatingwebhook.yaml\napiVersion: admissionregistration.k8s.io/v1beta1\nkind: MutatingWebhookConfiguration\nmetadata:\n name: istio-sidecar-injector\n namespace: istio-system\n labels:\n app: istio-sidecar-injector\n chart: sidecarInjectorWebhook-0.8.0\n release: RELEASE-NAME\n heritage: Tiller\nwebhooks:\n - name: sidecar-injector.istio.io\n clientConfig:\n service:\n name: istio-sidecar-injector\n namespace: istio-system\n path: \"/inject\"\n caBundle: \"\"\n rules:\n - operations: [ \"CREATE\" ]\n apiGroups: [\"\"]\n apiVersions: [\"v1\"]\n resources: [\"pods\"]\n failurePolicy: Fail\n namespaceSelector:\n matchLabels:\n istio-injection: enabled\n\n---\n# Source: istio/charts/grafana/templates/ingress.yaml\n\n---\n# Source: istio/charts/mixer/templates/config.yaml\n\n\n---\n# Source: istio/charts/prometheus/templates/ingress.yaml\n\n---\n# Source: istio/charts/servicegraph/templates/ingress.yaml\n\n---\n# Source: istio/charts/tracing/templates/ingress.yaml\n\n---\n# Source: istio/charts/tracing/templates/service-jaeger.yaml\n\n\n\n" }, { "path": "examples/90_Kubernetes/istio/rendered/istio-v1.0-minikube.yml", "content": "apiVersion: v1\nkind: Namespace\nmetadata:\n name: istio-system\n---\n# Source: istio/charts/galley/templates/configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio-galley-configuration\n namespace: istio-system\n labels:\n app: istio-galley\n chart: galley-1.0.0\n release: istio\n heritage: Tiller\n istio: mixer\ndata:\n validatingwebhookconfiguration.yaml: |- \n apiVersion: admissionregistration.k8s.io/v1beta1\n kind: ValidatingWebhookConfiguration\n metadata:\n name: istio-galley\n namespace: istio-system\n labels:\n app: istio-galley\n chart: galley-1.0.0\n release: istio\n heritage: Tiller\n webhooks:\n - name: pilot.validation.istio.io\n clientConfig:\n service:\n name: istio-galley\n namespace: istio-system\n path: \"/admitpilot\"\n caBundle: \"\"\n rules:\n - operations:\n - CREATE\n - UPDATE\n apiGroups:\n - config.istio.io\n apiVersions:\n - v1alpha2\n resources:\n - httpapispecs\n - httpapispecbindings\n - quotaspecs\n - quotaspecbindings\n - operations:\n - CREATE\n - UPDATE\n apiGroups:\n - rbac.istio.io\n apiVersions:\n - \"*\"\n resources:\n - \"*\"\n - operations:\n - CREATE\n - UPDATE\n apiGroups:\n - authentication.istio.io\n apiVersions:\n - \"*\"\n resources:\n - \"*\"\n - operations:\n - CREATE\n - UPDATE\n apiGroups:\n - networking.istio.io\n apiVersions:\n - \"*\"\n resources:\n - destinationrules\n - envoyfilters\n - gateways\n # disabled per @costinm's request\n # - serviceentries\n - virtualservices\n failurePolicy: Fail\n - name: mixer.validation.istio.io\n clientConfig:\n service:\n name: istio-galley\n namespace: istio-system\n path: \"/admitmixer\"\n caBundle: \"\"\n rules:\n - operations:\n - CREATE\n - UPDATE\n apiGroups:\n - config.istio.io\n apiVersions:\n - v1alpha2\n resources:\n - rules\n - attributemanifests\n - circonuses\n - deniers\n - fluentds\n - kubernetesenvs\n - listcheckers\n - memquotas\n - noops\n - opas\n - prometheuses\n - rbacs\n - servicecontrols\n - solarwindses\n - stackdrivers\n - statsds\n - stdios\n - apikeys\n - authorizations\n - checknothings\n # - kuberneteses\n - listentries\n - logentries\n - metrics\n - quotas\n - reportnothings\n - servicecontrolreports\n - tracespans\n failurePolicy: Fail\n\n\n---\n# Source: istio/charts/mixer/templates/configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio-statsd-prom-bridge\n namespace: istio-system\n labels:\n app: istio-statsd-prom-bridge\n chart: mixer-1.0.0\n release: istio\n heritage: Tiller\n istio: mixer\ndata:\n mapping.conf: |-\n\n---\n# Source: istio/charts/prometheus/templates/configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: prometheus\n namespace: istio-system\n labels:\n app: prometheus\n chart: prometheus-0.1.0\n release: istio\n heritage: Tiller\ndata:\n prometheus.yml: |-\n global:\n scrape_interval: 15s\n scrape_configs:\n\n - job_name: 'istio-mesh'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n\n kubernetes_sd_configs:\n - role: endpoints\n namespaces:\n names:\n - istio-system\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-telemetry;prometheus\n\n - job_name: 'envoy'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n namespaces:\n names:\n - istio-system\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-statsd-prom-bridge;statsd-prom\n\n - job_name: 'istio-policy'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n namespaces:\n names:\n - istio-system\n\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-policy;http-monitoring\n\n - job_name: 'istio-telemetry'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n namespaces:\n names:\n - istio-system\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-telemetry;http-monitoring\n\n - job_name: 'pilot'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n namespaces:\n names:\n - istio-system\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-pilot;http-monitoring\n\n - job_name: 'galley'\n # Override the global default and scrape targets from this job every 5 seconds.\n scrape_interval: 5s\n # metrics_path defaults to '/metrics'\n # scheme defaults to 'http'.\n\n kubernetes_sd_configs:\n - role: endpoints\n namespaces:\n names:\n - istio-system\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: istio-galley;http-monitoring\n\n # scrape config for API servers\n - job_name: 'kubernetes-apiservers'\n kubernetes_sd_configs:\n - role: endpoints\n namespaces:\n names:\n - default\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n action: keep\n regex: kubernetes;https\n\n # scrape config for nodes (kubelet)\n - job_name: 'kubernetes-nodes'\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n kubernetes_sd_configs:\n - role: node\n relabel_configs:\n - action: labelmap\n regex: __meta_kubernetes_node_label_(.+)\n - target_label: __address__\n replacement: kubernetes.default.svc:443\n - source_labels: [__meta_kubernetes_node_name]\n regex: (.+)\n target_label: __metrics_path__\n replacement: /api/v1/nodes/${1}/proxy/metrics\n\n # Scrape config for Kubelet cAdvisor.\n #\n # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics\n # (those whose names begin with 'container_') have been removed from the\n # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to\n # retrieve those metrics.\n #\n # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor\n # HTTP endpoint; use \"replacement: /api/v1/nodes/${1}:4194/proxy/metrics\"\n # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with\n # the --cadvisor-port=0 Kubelet flag).\n #\n # This job is not necessary and should be removed in Kubernetes 1.6 and\n # earlier versions, or it will cause the metrics to be scraped twice.\n - job_name: 'kubernetes-cadvisor'\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n kubernetes_sd_configs:\n - role: node\n relabel_configs:\n - action: labelmap\n regex: __meta_kubernetes_node_label_(.+)\n - target_label: __address__\n replacement: kubernetes.default.svc:443\n - source_labels: [__meta_kubernetes_node_name]\n regex: (.+)\n target_label: __metrics_path__\n replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor\n\n # scrape config for service endpoints.\n - job_name: 'kubernetes-service-endpoints'\n kubernetes_sd_configs:\n - role: endpoints\n relabel_configs:\n - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]\n action: replace\n target_label: __scheme__\n regex: (https?)\n - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]\n action: replace\n target_label: __metrics_path__\n regex: (.+)\n - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]\n action: replace\n target_label: __address__\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n - action: labelmap\n regex: __meta_kubernetes_service_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: kubernetes_namespace\n - source_labels: [__meta_kubernetes_service_name]\n action: replace\n target_label: kubernetes_name\n\n # Example scrape config for pods\n - job_name: 'kubernetes-pods'\n kubernetes_sd_configs:\n - role: pod\n\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]\n action: replace\n target_label: __metrics_path__\n regex: (.+)\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]\n action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod_name\n\n---\n# Source: istio/charts/security/templates/configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio-security-custom-resources\n namespace: istio-system\n labels:\n app: istio-security\n chart: security-1.0.0\n release: istio\n heritage: Tiller\n istio: security\ndata:\n custom-resources.yaml: |-\n run.sh: |- \n #!/bin/sh\n \n set -x\n \n if [ \"$#\" -ne \"1\" ]; then\n echo \"first argument should be path to custom resource yaml\"\n exit 1\n fi\n \n pathToResourceYAML=${1}\n \n /kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null\n if [ \"$?\" -eq 0 ]; then\n echo \"istio-galley validatingwebhookconfiguration found - waiting for istio-galley deployment to be ready\"\n while true; do\n /kubectl -n istio-system get deployment istio-galley 2>/dev/null\n if [ \"$?\" -eq 0 ]; then\n break\n fi\n sleep 1\n done\n /kubectl -n istio-system rollout status deployment istio-galley\n if [ \"$?\" -ne 0 ]; then\n echo \"istio-galley deployment rollout status check failed\"\n exit 1\n fi\n echo \"istio-galley deployment ready for configuration validation\"\n fi\n sleep 5\n /kubectl apply -f ${pathToResourceYAML}\n \n\n---\n# Source: istio/templates/configmap.yaml\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio\n namespace: istio-system\n labels:\n app: istio\n chart: istio-1.0.0\n release: istio\n heritage: Tiller\ndata:\n mesh: |-\n # Set the following variable to true to disable policy checks by the Mixer.\n # Note that metrics will still be reported to the Mixer.\n disablePolicyChecks: false\n\n # Set enableTracing to false to disable request tracing.\n enableTracing: true\n\n # Set accessLogFile to empty string to disable access log.\n accessLogFile: \"/dev/stdout\"\n #\n # Deprecated: mixer is using EDS\n mixerCheckServer: istio-policy.istio-system.svc.cluster.local:9091\n mixerReportServer: istio-telemetry.istio-system.svc.cluster.local:9091\n\n # Unix Domain Socket through which envoy communicates with NodeAgent SDS to get\n # key/cert for mTLS. Use secret-mount files instead of SDS if set to empty. \n sdsUdsPath: \"\"\n \n # How frequently should Envoy fetch key/cert from NodeAgent.\n sdsRefreshDelay: 15s\n\n #\n defaultConfig:\n #\n # TCP connection timeout between Envoy & the application, and between Envoys.\n connectTimeout: 10s\n #\n ### ADVANCED SETTINGS #############\n # Where should envoy's configuration be stored in the istio-proxy container\n configPath: \"/etc/istio/proxy\"\n binaryPath: \"/usr/local/bin/envoy\"\n # The pseudo service name used for Envoy.\n serviceCluster: istio-proxy\n # These settings that determine how long an old Envoy\n # process should be kept alive after an occasional reload.\n drainDuration: 45s\n parentShutdownDuration: 1m0s\n #\n # The mode used to redirect inbound connections to Envoy. This setting\n # has no effect on outbound traffic: iptables REDIRECT is always used for\n # outbound connections.\n # If \"REDIRECT\", use iptables REDIRECT to NAT and redirect to Envoy.\n # The \"REDIRECT\" mode loses source addresses during redirection.\n # If \"TPROXY\", use iptables TPROXY to redirect to Envoy.\n # The \"TPROXY\" mode preserves both the source and destination IP\n # addresses and ports, so that they can be used for advanced filtering\n # and manipulation.\n # The \"TPROXY\" mode also configures the sidecar to run with the\n # CAP_NET_ADMIN capability, which is required to use TPROXY.\n #interceptionMode: REDIRECT\n #\n # Port where Envoy listens (on local host) for admin commands\n # You can exec into the istio-proxy container in a pod and\n # curl the admin port (curl http://localhost:15000/) to obtain\n # diagnostic information from Envoy. See\n # https://lyft.github.io/envoy/docs/operations/admin.html\n # for more details\n proxyAdminPort: 15000\n #\n # Zipkin trace collector\n zipkinAddress: zipkin.istio-system:9411\n #\n # Statsd metrics collector converts statsd metrics into Prometheus metrics.\n statsdUdpAddress: istio-statsd-prom-bridge.istio-system:9125\n #\n # Mutual TLS authentication between sidecars and istio control plane.\n controlPlaneAuthPolicy: NONE\n #\n # Address where istio Pilot service is running\n discoveryAddress: istio-pilot.istio-system:15007\n\n---\n# Source: istio/templates/sidecar-injector-configmap.yaml\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: istio-sidecar-injector\n namespace: istio-system\n labels:\n app: istio\n chart: istio-1.0.0\n release: istio\n heritage: Tiller\n istio: sidecar-injector\ndata:\n config: |-\n policy: enabled\n template: |-\n initContainers:\n - name: istio-init\n image: \"docker.io/istio/proxy_init:1.0.0\"\n args:\n - \"-p\"\n - [[ .MeshConfig.ProxyListenPort ]]\n - \"-u\"\n - 1337\n - \"-m\"\n - [[ or (index .ObjectMeta.Annotations \"sidecar.istio.io/interceptionMode\") .ProxyConfig.InterceptionMode.String ]]\n - \"-i\"\n [[ if (isset .ObjectMeta.Annotations \"traffic.sidecar.istio.io/includeOutboundIPRanges\") -]]\n - \"[[ index .ObjectMeta.Annotations \"traffic.sidecar.istio.io/includeOutboundIPRanges\" ]]\"\n [[ else -]]\n - \"*\"\n [[ end -]]\n - \"-x\"\n [[ if (isset .ObjectMeta.Annotations \"traffic.sidecar.istio.io/excludeOutboundIPRanges\") -]]\n - \"[[ index .ObjectMeta.Annotations \"traffic.sidecar.istio.io/excludeOutboundIPRanges\" ]]\"\n [[ else -]]\n - \"\"\n [[ end -]]\n - \"-b\"\n [[ if (isset .ObjectMeta.Annotations \"traffic.sidecar.istio.io/includeInboundPorts\") -]]\n - \"[[ index .ObjectMeta.Annotations \"traffic.sidecar.istio.io/includeInboundPorts\" ]]\"\n [[ else -]]\n - [[ range .Spec.Containers -]][[ range .Ports -]][[ .ContainerPort -]], [[ end -]][[ end -]][[ end]]\n - \"-d\"\n [[ if (isset .ObjectMeta.Annotations \"traffic.sidecar.istio.io/excludeInboundPorts\") -]]\n - \"[[ index .ObjectMeta.Annotations \"traffic.sidecar.istio.io/excludeInboundPorts\" ]]\"\n [[ else -]]\n - \"\"\n [[ end -]]\n imagePullPolicy: IfNotPresent\n securityContext:\n capabilities:\n add:\n - NET_ADMIN\n privileged: true\n restartPolicy: Always\n \n containers:\n - name: istio-proxy\n image: [[ if (isset .ObjectMeta.Annotations \"sidecar.istio.io/proxyImage\") -]]\n \"[[ index .ObjectMeta.Annotations \"sidecar.istio.io/proxyImage\" ]]\"\n [[ else -]]\n docker.io/istio/proxyv2:1.0.0\n [[ end -]]\n args:\n - proxy\n - sidecar\n - --configPath\n - [[ .ProxyConfig.ConfigPath ]]\n - --binaryPath\n - [[ .ProxyConfig.BinaryPath ]]\n - --serviceCluster\n [[ if ne \"\" (index .ObjectMeta.Labels \"app\") -]]\n - [[ index .ObjectMeta.Labels \"app\" ]]\n [[ else -]]\n - \"istio-proxy\"\n [[ end -]]\n - --drainDuration\n - [[ formatDuration .ProxyConfig.DrainDuration ]]\n - --parentShutdownDuration\n - [[ formatDuration .ProxyConfig.ParentShutdownDuration ]]\n - --discoveryAddress\n - [[ .ProxyConfig.DiscoveryAddress ]]\n - --discoveryRefreshDelay\n - [[ formatDuration .ProxyConfig.DiscoveryRefreshDelay ]]\n - --zipkinAddress\n - [[ .ProxyConfig.ZipkinAddress ]]\n - --connectTimeout\n - [[ formatDuration .ProxyConfig.ConnectTimeout ]]\n - --statsdUdpAddress\n - [[ .ProxyConfig.StatsdUdpAddress ]]\n - --proxyAdminPort\n - [[ .ProxyConfig.ProxyAdminPort ]]\n - --controlPlaneAuthPolicy\n - [[ or (index .ObjectMeta.Annotations \"sidecar.istio.io/controlPlaneAuthPolicy\") .ProxyConfig.ControlPlaneAuthPolicy ]]\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: ISTIO_META_POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: ISTIO_META_INTERCEPTION_MODE\n value: [[ or (index .ObjectMeta.Annotations \"sidecar.istio.io/interceptionMode\") .ProxyConfig.InterceptionMode.String ]]\n imagePullPolicy: IfNotPresent\n securityContext:\n privileged: false\n readOnlyRootFilesystem: true\n [[ if eq (or (index .ObjectMeta.Annotations \"sidecar.istio.io/interceptionMode\") .ProxyConfig.InterceptionMode.String) \"TPROXY\" -]]\n capabilities:\n add:\n - NET_ADMIN\n runAsGroup: 1337\n [[ else -]]\n runAsUser: 1337\n [[ end -]]\n restartPolicy: Always\n resources:\n [[ if (isset .ObjectMeta.Annotations \"sidecar.istio.io/proxyCPU\") -]]\n requests:\n cpu: \"[[ index .ObjectMeta.Annotations \"sidecar.istio.io/proxyCPU\" ]]\"\n memory: \"[[ index .ObjectMeta.Annotations \"sidecar.istio.io/proxyMemory\" ]]\"\n [[ else -]]\n requests:\n cpu: 10m\n \n [[ end -]]\n volumeMounts:\n - mountPath: /etc/istio/proxy\n name: istio-envoy\n - mountPath: /etc/certs/\n name: istio-certs\n readOnly: true\n volumes:\n - emptyDir:\n medium: Memory\n name: istio-envoy\n - name: istio-certs\n secret:\n optional: true\n [[ if eq .Spec.ServiceAccountName \"\" -]]\n secretName: istio.default\n [[ else -]]\n secretName: [[ printf \"istio.%s\" .Spec.ServiceAccountName ]]\n [[ end -]]\n\n---\n# Source: istio/charts/galley/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-galley-service-account\n namespace: istio-system\n labels:\n app: istio-galley\n chart: galley-1.0.0\n heritage: Tiller\n release: istio\n\n---\n# Source: istio/charts/gateways/templates/serviceaccount.yaml\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-egressgateway-service-account\n namespace: istio-system\n labels:\n app: egressgateway\n chart: gateways-1.0.0\n heritage: Tiller\n release: istio\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-ingressgateway-service-account\n namespace: istio-system\n labels:\n app: ingressgateway\n chart: gateways-1.0.0\n heritage: Tiller\n release: istio\n---\n\n---\n# Source: istio/charts/mixer/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-mixer-service-account\n namespace: istio-system\n labels:\n app: mixer\n chart: mixer-1.0.0\n heritage: Tiller\n release: istio\n\n---\n# Source: istio/charts/pilot/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-pilot-service-account\n namespace: istio-system\n labels:\n app: istio-pilot\n chart: pilot-1.0.0\n heritage: Tiller\n release: istio\n\n---\n# Source: istio/charts/prometheus/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: prometheus\n namespace: istio-system\n\n---\n# Source: istio/charts/security/templates/cleanup-secrets.yaml\n# The reason for creating a ServiceAccount and ClusterRole specifically for this\n# post-delete hooked job is because the citadel ServiceAccount is being deleted\n# before this hook is launched. On the other hand, running this hook before the\n# deletion of the citadel (e.g. pre-delete) won't delete the secrets because they\n# will be re-created immediately by the to-be-deleted citadel.\n#\n# It's also important that the ServiceAccount, ClusterRole and ClusterRoleBinding\n# will be ready before running the hooked Job therefore the hook weights.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-cleanup-secrets-service-account\n namespace: istio-system\n annotations:\n \"helm.sh/hook\": post-delete\n \"helm.sh/hook-delete-policy\": hook-succeeded\n \"helm.sh/hook-weight\": \"1\"\n labels:\n app: security\n chart: security-1.0.0\n heritage: Tiller\n release: istio\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-cleanup-secrets-istio-system\n annotations:\n \"helm.sh/hook\": post-delete\n \"helm.sh/hook-delete-policy\": hook-succeeded\n \"helm.sh/hook-weight\": \"1\"\n labels:\n app: security\n chart: security-1.0.0\n heritage: Tiller\n release: istio\nrules:\n- apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"list\", \"delete\"]\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-cleanup-secrets-istio-system\n annotations:\n \"helm.sh/hook\": post-delete\n \"helm.sh/hook-delete-policy\": hook-succeeded\n \"helm.sh/hook-weight\": \"2\"\n labels:\n app: security\n chart: security-1.0.0\n heritage: Tiller\n release: istio\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-cleanup-secrets-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-cleanup-secrets-service-account\n namespace: istio-system\n---\napiVersion: batch/v1\nkind: Job\nmetadata:\n name: istio-cleanup-secrets\n namespace: istio-system\n annotations:\n \"helm.sh/hook\": post-delete\n \"helm.sh/hook-delete-policy\": hook-succeeded\n \"helm.sh/hook-weight\": \"3\"\n labels:\n app: security\n chart: security-1.0.0\n release: istio\n heritage: Tiller\nspec:\n template:\n metadata:\n name: istio-cleanup-secrets\n labels:\n app: security\n release: istio\n spec:\n serviceAccountName: istio-cleanup-secrets-service-account\n containers:\n - name: hyperkube\n image: \"quay.io/coreos/hyperkube:v1.7.6_coreos.0\"\n command:\n - /bin/bash\n - -c\n - >\n kubectl get secret --all-namespaces | grep \"istio.io/key-and-cert\" | while read -r entry; do\n ns=$(echo $entry | awk '{print $1}');\n name=$(echo $entry | awk '{print $2}');\n kubectl delete secret $name -n $ns;\n done\n restartPolicy: OnFailure\n\n---\n# Source: istio/charts/security/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-citadel-service-account\n namespace: istio-system\n labels:\n app: security\n chart: security-1.0.0\n heritage: Tiller\n release: istio\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: istio-sidecar-injector-service-account\n namespace: istio-system\n labels:\n app: istio-sidecar-injector\n chart: sidecarInjectorWebhook-1.0.0\n heritage: Tiller\n release: istio\n\n---\n# Source: istio/templates/crds.yaml\n# \n# these CRDs only make sense when pilot is enabled\n#\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: virtualservices.networking.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: VirtualService\n listKind: VirtualServiceList\n plural: virtualservices\n singular: virtualservice\n categories:\n - istio-io\n - networking-istio-io\n scope: Namespaced\n version: v1alpha3\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: destinationrules.networking.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: DestinationRule\n listKind: DestinationRuleList\n plural: destinationrules\n singular: destinationrule\n categories:\n - istio-io\n - networking-istio-io\n scope: Namespaced\n version: v1alpha3\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: serviceentries.networking.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: ServiceEntry\n listKind: ServiceEntryList\n plural: serviceentries\n singular: serviceentry\n categories:\n - istio-io\n - networking-istio-io\n scope: Namespaced\n version: v1alpha3\n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: gateways.networking.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n \"helm.sh/hook-weight\": \"-5\"\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: Gateway\n plural: gateways\n singular: gateway\n categories:\n - istio-io\n - networking-istio-io\n scope: Namespaced\n version: v1alpha3 \n---\napiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition\nmetadata:\n name: envoyfilters.networking.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: istio-pilot\nspec:\n group: networking.istio.io\n names:\n kind: EnvoyFilter\n plural: envoyfilters\n singular: envoyfilter\n categories:\n - istio-io\n - networking-istio-io\n scope: Namespaced\n version: v1alpha3\n---\n#\n\n# these CRDs only make sense when security is enabled\n#\n\n#\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n annotations:\n \"helm.sh/hook\": crd-install\n name: httpapispecbindings.config.istio.io\nspec:\n group: config.istio.io\n names:\n kind: HTTPAPISpecBinding\n plural: httpapispecbindings\n singular: httpapispecbinding\n categories:\n - istio-io\n - apim-istio-io\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n annotations:\n \"helm.sh/hook\": crd-install\n name: httpapispecs.config.istio.io\nspec:\n group: config.istio.io\n names:\n kind: HTTPAPISpec\n plural: httpapispecs\n singular: httpapispec\n categories:\n - istio-io\n - apim-istio-io\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n annotations:\n \"helm.sh/hook\": crd-install\n name: quotaspecbindings.config.istio.io\nspec:\n group: config.istio.io\n names:\n kind: QuotaSpecBinding\n plural: quotaspecbindings\n singular: quotaspecbinding\n categories:\n - istio-io\n - apim-istio-io\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n annotations:\n \"helm.sh/hook\": crd-install\n name: quotaspecs.config.istio.io\nspec:\n group: config.istio.io\n names:\n kind: QuotaSpec\n plural: quotaspecs\n singular: quotaspec\n categories:\n - istio-io\n - apim-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\n# Mixer CRDs\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: rules.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: istio.io.mixer\n istio: core\nspec:\n group: config.istio.io\n names:\n kind: rule\n plural: rules\n singular: rule\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: attributemanifests.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: istio.io.mixer\n istio: core\nspec:\n group: config.istio.io\n names:\n kind: attributemanifest\n plural: attributemanifests\n singular: attributemanifest\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: bypasses.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: bypass\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: bypass\n plural: bypasses\n singular: bypass\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: circonuses.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: circonus\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: circonus\n plural: circonuses\n singular: circonus\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: deniers.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: denier\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: denier\n plural: deniers\n singular: denier\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: fluentds.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: fluentd\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: fluentd\n plural: fluentds\n singular: fluentd\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: kubernetesenvs.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: kubernetesenv\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: kubernetesenv\n plural: kubernetesenvs\n singular: kubernetesenv\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: listcheckers.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: listchecker\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: listchecker\n plural: listcheckers\n singular: listchecker\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: memquotas.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: memquota\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: memquota\n plural: memquotas\n singular: memquota\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: noops.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: noop\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: noop\n plural: noops\n singular: noop\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: opas.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: opa\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: opa\n plural: opas\n singular: opa\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: prometheuses.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: prometheus\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: prometheus\n plural: prometheuses\n singular: prometheus\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: rbacs.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: rbac\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: rbac\n plural: rbacs\n singular: rbac\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: redisquotas.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n package: redisquota\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: redisquota\n plural: redisquotas\n singular: redisquota\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: servicecontrols.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: servicecontrol\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: servicecontrol\n plural: servicecontrols\n singular: servicecontrol\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: signalfxs.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: signalfx\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: signalfx\n plural: signalfxs\n singular: signalfx\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: solarwindses.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: solarwinds\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: solarwinds\n plural: solarwindses\n singular: solarwinds\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: stackdrivers.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: stackdriver\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: stackdriver\n plural: stackdrivers\n singular: stackdriver\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: statsds.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: statsd\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: statsd\n plural: statsds\n singular: statsd\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: stdios.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: stdio\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: stdio\n plural: stdios\n singular: stdio\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: apikeys.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: apikey\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: apikey\n plural: apikeys\n singular: apikey\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: authorizations.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: authorization\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: authorization\n plural: authorizations\n singular: authorization\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: checknothings.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: checknothing\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: checknothing\n plural: checknothings\n singular: checknothing\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: kuberneteses.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: adapter.template.kubernetes\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: kubernetes\n plural: kuberneteses\n singular: kubernetes\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: listentries.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: listentry\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: listentry\n plural: listentries\n singular: listentry\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: logentries.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: logentry\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: logentry\n plural: logentries\n singular: logentry\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: edges.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: edge\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: edge\n plural: edges\n singular: edge\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: metrics.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: metric\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: metric\n plural: metrics\n singular: metric\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: quotas.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: quota\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: quota\n plural: quotas\n singular: quota\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: reportnothings.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: reportnothing\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: reportnothing\n plural: reportnothings\n singular: reportnothing\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: servicecontrolreports.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: servicecontrolreport\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: servicecontrolreport\n plural: servicecontrolreports\n singular: servicecontrolreport\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: tracespans.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: tracespan\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: tracespan\n plural: tracespans\n singular: tracespan\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: rbacconfigs.rbac.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: istio.io.mixer\n istio: rbac\nspec:\n group: rbac.istio.io\n names:\n kind: RbacConfig\n plural: rbacconfigs\n singular: rbacconfig\n categories:\n - istio-io\n - rbac-istio-io\n scope: Namespaced\n version: v1alpha1\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: serviceroles.rbac.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: istio.io.mixer\n istio: rbac\nspec:\n group: rbac.istio.io\n names:\n kind: ServiceRole\n plural: serviceroles\n singular: servicerole\n categories:\n - istio-io\n - rbac-istio-io\n scope: Namespaced\n version: v1alpha1\n---\n\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: servicerolebindings.rbac.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: istio.io.mixer\n istio: rbac\nspec:\n group: rbac.istio.io\n names:\n kind: ServiceRoleBinding\n plural: servicerolebindings\n singular: servicerolebinding\n categories:\n - istio-io\n - rbac-istio-io\n scope: Namespaced\n version: v1alpha1\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: adapters.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: adapter\n istio: mixer-adapter\nspec:\n group: config.istio.io\n names:\n kind: adapter\n plural: adapters\n singular: adapter\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: instances.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: instance\n istio: mixer-instance\nspec:\n group: config.istio.io\n names:\n kind: instance\n plural: instances\n singular: instance\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: templates.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: template\n istio: mixer-template\nspec:\n group: config.istio.io\n names:\n kind: template\n plural: templates\n singular: template\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\nkind: CustomResourceDefinition\napiVersion: apiextensions.k8s.io/v1beta1\nmetadata:\n name: handlers.config.istio.io\n annotations:\n \"helm.sh/hook\": crd-install\n labels:\n app: mixer\n package: handler\n istio: mixer-handler\nspec:\n group: config.istio.io\n names:\n kind: handler\n plural: handlers\n singular: handler\n categories:\n - istio-io\n - policy-istio-io\n scope: Namespaced\n version: v1alpha2\n---\n#\n# \n---\n# Source: istio/charts/galley/templates/clusterrole.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-galley-istio-system\n labels:\n app: istio-galley\n chart: galley-1.0.0\n heritage: Tiller\n release: istio\nrules:\n- apiGroups: [\"admissionregistration.k8s.io\"]\n resources: [\"validatingwebhookconfigurations\"]\n verbs: [\"*\"]\n- apiGroups: [\"config.istio.io\"] # istio mixer CRD watcher\n resources: [\"*\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"*\"]\n resources: [\"deployments\"]\n resourceNames: [\"istio-galley\"]\n verbs: [\"get\"]\n\n---\n# Source: istio/charts/gateways/templates/clusterrole.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n labels:\n app: gateways\n chart: gateways-1.0.0\n heritage: Tiller\n release: istio\n name: istio-egressgateway-istio-system\nrules:\n- apiGroups: [\"extensions\"]\n resources: [\"thirdpartyresources\", \"virtualservices\", \"destinationrules\", \"gateways\"]\n verbs: [\"get\", \"watch\", \"list\", \"update\"]\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n labels:\n app: gateways\n chart: gateways-1.0.0\n heritage: Tiller\n release: istio\n name: istio-ingressgateway-istio-system\nrules:\n- apiGroups: [\"extensions\"]\n resources: [\"thirdpartyresources\", \"virtualservices\", \"destinationrules\", \"gateways\"]\n verbs: [\"get\", \"watch\", \"list\", \"update\"]\n---\n\n---\n# Source: istio/charts/mixer/templates/clusterrole.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-mixer-istio-system\n labels:\n app: mixer\n chart: mixer-1.0.0\n heritage: Tiller\n release: istio\nrules:\n- apiGroups: [\"config.istio.io\"] # istio CRD watcher\n resources: [\"*\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"patch\"]\n- apiGroups: [\"rbac.istio.io\"] # istio RBAC watcher\n resources: [\"*\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources: [\"configmaps\", \"endpoints\", \"pods\", \"services\", \"namespaces\", \"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"extensions\"]\n resources: [\"replicasets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"apps\"]\n resources: [\"replicasets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n\n---\n# Source: istio/charts/pilot/templates/clusterrole.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-pilot-istio-system\n labels:\n app: istio-pilot\n chart: pilot-1.0.0\n heritage: Tiller\n release: istio\nrules:\n- apiGroups: [\"config.istio.io\"]\n resources: [\"*\"]\n verbs: [\"*\"]\n- apiGroups: [\"rbac.istio.io\"]\n resources: [\"*\"]\n verbs: [\"get\", \"watch\", \"list\"]\n- apiGroups: [\"networking.istio.io\"]\n resources: [\"*\"]\n verbs: [\"*\"]\n- apiGroups: [\"authentication.istio.io\"]\n resources: [\"*\"]\n verbs: [\"*\"]\n- apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"*\"]\n- apiGroups: [\"extensions\"]\n resources: [\"thirdpartyresources\", \"thirdpartyresources.extensions\", \"ingresses\", \"ingresses/status\"]\n verbs: [\"*\"]\n- apiGroups: [\"\"]\n resources: [\"configmaps\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\"]\n- apiGroups: [\"\"]\n resources: [\"endpoints\", \"pods\", \"services\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources: [\"namespaces\", \"nodes\", \"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n\n---\n# Source: istio/charts/prometheus/templates/clusterrole.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: prometheus-istio-system\nrules:\n- apiGroups: [\"\"]\n resources:\n - nodes\n - services\n - endpoints\n - pods\n - nodes/proxy\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources:\n - configmaps\n verbs: [\"get\"]\n- nonResourceURLs: [\"/metrics\"]\n verbs: [\"get\"]\n\n---\n# Source: istio/charts/security/templates/clusterrole.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-citadel-istio-system\n labels:\n app: security\n chart: security-1.0.0\n heritage: Tiller\n release: istio\nrules:\n- apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"create\", \"get\", \"watch\", \"list\", \"update\", \"delete\"]\n- apiGroups: [\"\"]\n resources: [\"serviceaccounts\"]\n verbs: [\"get\", \"watch\", \"list\"]\n- apiGroups: [\"\"]\n resources: [\"services\"]\n verbs: [\"get\", \"watch\", \"list\"]\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/clusterrole.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: istio-sidecar-injector-istio-system\n labels:\n app: istio-sidecar-injector\n chart: sidecarInjectorWebhook-1.0.0\n heritage: Tiller\n release: istio\nrules:\n- apiGroups: [\"*\"]\n resources: [\"configmaps\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"admissionregistration.k8s.io\"]\n resources: [\"mutatingwebhookconfigurations\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n\n---\n# Source: istio/charts/galley/templates/clusterrolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-galley-admin-role-binding-istio-system\n labels:\n app: istio-galley\n chart: galley-1.0.0\n heritage: Tiller\n release: istio\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-galley-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-galley-service-account\n namespace: istio-system\n\n---\n# Source: istio/charts/gateways/templates/clusterrolebindings.yaml\n\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-egressgateway-istio-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-egressgateway-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-egressgateway-service-account\n namespace: istio-system\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-ingressgateway-istio-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-ingressgateway-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-ingressgateway-service-account\n namespace: istio-system\n---\n\n---\n# Source: istio/charts/mixer/templates/clusterrolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-mixer-admin-role-binding-istio-system\n labels:\n app: mixer\n chart: mixer-1.0.0\n heritage: Tiller\n release: istio\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-mixer-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-mixer-service-account\n namespace: istio-system\n\n---\n# Source: istio/charts/pilot/templates/clusterrolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-pilot-istio-system\n labels:\n app: istio-pilot\n chart: pilot-1.0.0\n heritage: Tiller\n release: istio\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-pilot-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-pilot-service-account\n namespace: istio-system\n\n---\n# Source: istio/charts/prometheus/templates/clusterrolebindings.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: prometheus-istio-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: prometheus-istio-system\nsubjects:\n- kind: ServiceAccount\n name: prometheus\n namespace: istio-system\n\n---\n# Source: istio/charts/security/templates/clusterrolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-citadel-istio-system\n labels:\n app: security\n chart: security-1.0.0\n heritage: Tiller\n release: istio\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-citadel-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-citadel-service-account\n namespace: istio-system\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/clusterrolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: istio-sidecar-injector-admin-role-binding-istio-system\n labels:\n app: istio-sidecar-injector\n chart: sidecarInjectorWebhook-1.0.0\n heritage: Tiller\n release: istio\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: istio-sidecar-injector-istio-system\nsubjects:\n - kind: ServiceAccount\n name: istio-sidecar-injector-service-account\n namespace: istio-system\n\n---\n# Source: istio/charts/galley/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-galley\n namespace: istio-system\n labels:\n istio: galley\nspec:\n ports:\n - port: 443\n name: https-validation\n - port: 9093\n name: http-monitoring\n selector:\n istio: galley\n\n---\n# Source: istio/charts/gateways/templates/service.yaml\n\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-egressgateway\n namespace: istio-system\n annotations:\n labels:\n chart: gateways-1.0.0\n release: istio\n heritage: Tiller\n app: istio-egressgateway\n istio: egressgateway\nspec:\n type: NodePort\n selector:\n app: istio-egressgateway\n istio: egressgateway\n ports:\n -\n name: http2\n port: 80\n -\n name: https\n port: 443\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-ingressgateway\n namespace: istio-system\n annotations:\n labels:\n chart: gateways-1.0.0\n release: istio\n heritage: Tiller\n app: istio-ingressgateway\n istio: ingressgateway\nspec:\n type: NodePort\n selector:\n app: istio-ingressgateway\n istio: ingressgateway\n ports:\n -\n name: http2\n nodePort: 31380\n port: 80\n targetPort: 80\n -\n name: https\n nodePort: 31390\n port: 443\n -\n name: tcp\n nodePort: 31400\n port: 31400\n -\n name: tcp-pilot-grpc-tls\n port: 15011\n targetPort: 15011\n -\n name: tcp-citadel-grpc-tls\n port: 8060\n targetPort: 8060\n -\n name: http2-prometheus\n port: 15030\n targetPort: 15030\n -\n name: http2-grafana\n port: 15031\n targetPort: 15031\n---\n\n---\n# Source: istio/charts/mixer/templates/service.yaml\n\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-policy\n namespace: istio-system\n labels:\n chart: mixer-1.0.0\n release: istio\n istio: mixer\nspec:\n ports:\n - name: grpc-mixer\n port: 9091\n - name: grpc-mixer-mtls\n port: 15004\n - name: http-monitoring\n port: 9093\n selector:\n istio: mixer\n istio-mixer-type: policy\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-telemetry\n namespace: istio-system\n labels:\n chart: mixer-1.0.0\n release: istio\n istio: mixer\nspec:\n ports:\n - name: grpc-mixer\n port: 9091\n - name: grpc-mixer-mtls\n port: 15004\n - name: http-monitoring\n port: 9093\n - name: prometheus\n port: 42422\n selector:\n istio: mixer\n istio-mixer-type: telemetry\n---\n\n---\n# Source: istio/charts/mixer/templates/statsdtoprom.yaml\n\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-statsd-prom-bridge\n namespace: istio-system\n labels:\n chart: mixer-1.0.0\n release: istio\n istio: statsd-prom-bridge\nspec:\n ports:\n - name: statsd-prom\n port: 9102\n - name: statsd-udp\n port: 9125\n protocol: UDP\n selector:\n istio: statsd-prom-bridge\n\n---\n\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-statsd-prom-bridge\n namespace: istio-system\n labels:\n chart: mixer-1.0.0\n release: istio\n istio: mixer\nspec:\n template:\n metadata:\n labels:\n istio: statsd-prom-bridge\n annotations:\n sidecar.istio.io/inject: \"false\"\n spec:\n serviceAccountName: istio-mixer-service-account\n volumes:\n - name: config-volume\n configMap:\n name: istio-statsd-prom-bridge\n containers:\n - name: statsd-prom-bridge\n image: \"docker.io/prom/statsd-exporter:v0.6.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9102\n - containerPort: 9125\n protocol: UDP\n args:\n - '-statsd.mapping-config=/etc/statsd/mapping.conf'\n resources:\n requests:\n cpu: 10m\n \n volumeMounts:\n - name: config-volume\n mountPath: /etc/statsd\n\n---\n# Source: istio/charts/pilot/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-pilot\n namespace: istio-system\n labels:\n app: istio-pilot\n chart: pilot-1.0.0\n release: istio\n heritage: Tiller\nspec:\n ports:\n - port: 15010\n name: grpc-xds # direct\n - port: 15011\n name: https-xds # mTLS\n - port: 8080\n name: http-legacy-discovery # direct\n - port: 9093\n name: http-monitoring\n selector:\n istio: pilot\n\n---\n# Source: istio/charts/prometheus/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: prometheus\n namespace: istio-system\n annotations:\n prometheus.io/scrape: 'true'\n labels:\n name: prometheus\nspec:\n selector:\n app: prometheus\n ports:\n - name: http-prometheus\n protocol: TCP\n port: 9090\n\n---\n# Source: istio/charts/security/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n # we use the normal name here (e.g. 'prometheus')\n # as grafana is configured to use this as a data source\n name: istio-citadel\n namespace: istio-system\n labels:\n app: istio-citadel\nspec:\n ports:\n - name: grpc-citadel\n port: 8060\n targetPort: 8060\n protocol: TCP\n - name: http-monitoring\n port: 9093\n selector:\n istio: citadel\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: istio-sidecar-injector\n namespace: istio-system\n labels:\n istio: sidecar-injector\nspec:\n ports:\n - port: 443\n selector:\n istio: sidecar-injector\n\n---\n# Source: istio/charts/galley/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-galley\n namespace: istio-system\n labels:\n app: galley\n chart: galley-1.0.0\n release: istio\n heritage: Tiller\n istio: galley\nspec:\n replicas: 1\n strategy:\n rollingUpdate:\n maxSurge: 1\n maxUnavailable: 0\n template:\n metadata:\n labels:\n istio: galley\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: istio-galley-service-account\n containers:\n - name: validator\n image: \"docker.io/istio/galley:1.0.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 443\n - containerPort: 9093\n command:\n - /usr/local/bin/galley\n - validator\n - --deployment-namespace=istio-system\n - --caCertFile=/etc/istio/certs/root-cert.pem\n - --tlsCertFile=/etc/istio/certs/cert-chain.pem\n - --tlsKeyFile=/etc/istio/certs/key.pem\n - --healthCheckInterval=2s\n - --healthCheckFile=/health\n - --webhook-config-file\n - /etc/istio/config/validatingwebhookconfiguration.yaml\n volumeMounts:\n - name: certs\n mountPath: /etc/istio/certs\n readOnly: true\n - name: config\n mountPath: /etc/istio/config\n readOnly: true\n livenessProbe:\n exec:\n command:\n - /usr/local/bin/galley\n - probe\n - --probe-path=/health\n - --interval=4s\n initialDelaySeconds: 4\n periodSeconds: 4\n readinessProbe:\n exec:\n command:\n - /usr/local/bin/galley\n - probe\n - --probe-path=/health\n - --interval=4s\n initialDelaySeconds: 4\n periodSeconds: 4\n resources:\n requests:\n cpu: 10m\n \n volumes:\n - name: certs\n secret:\n secretName: istio.istio-galley-service-account\n - name: config\n configMap:\n name: istio-galley-configuration\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/gateways/templates/deployment.yaml\n\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-egressgateway\n namespace: istio-system\n labels:\n app: egressgateway\n chart: gateways-1.0.0\n release: istio\n heritage: Tiller\n app: istio-egressgateway\n istio: egressgateway\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n app: istio-egressgateway\n istio: egressgateway\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: istio-egressgateway-service-account\n containers:\n - name: egressgateway\n image: \"docker.io/istio/proxyv2:1.0.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 80\n - containerPort: 443\n args:\n - proxy\n - router\n - -v\n - \"2\"\n - --discoveryRefreshDelay\n - '1s' #discoveryRefreshDelay\n - --drainDuration\n - '45s' #drainDuration\n - --parentShutdownDuration\n - '1m0s' #parentShutdownDuration\n - --connectTimeout\n - '10s' #connectTimeout\n - --serviceCluster\n - istio-egressgateway\n - --zipkinAddress\n - zipkin:9411\n - --statsdUdpAddress\n - istio-statsd-prom-bridge:9125\n - --proxyAdminPort\n - \"15000\"\n - --controlPlaneAuthPolicy\n - NONE\n - --discoveryAddress\n - istio-pilot.istio-system:8080\n resources:\n requests:\n cpu: 10m\n \n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n - name: ISTIO_META_POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n - name: egressgateway-certs\n mountPath: \"/etc/istio/egressgateway-certs\"\n readOnly: true\n - name: egressgateway-ca-certs\n mountPath: \"/etc/istio/egressgateway-ca-certs\"\n readOnly: true\n volumes:\n - name: istio-certs\n secret:\n secretName: istio.istio-egressgateway-service-account\n optional: true\n - name: egressgateway-certs\n secret:\n secretName: \"istio-egressgateway-certs\"\n optional: true\n - name: egressgateway-ca-certs\n secret:\n secretName: \"istio-egressgateway-ca-certs\"\n optional: true\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n---\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-ingressgateway\n namespace: istio-system\n labels:\n app: ingressgateway\n chart: gateways-1.0.0\n release: istio\n heritage: Tiller\n app: istio-ingressgateway\n istio: ingressgateway\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n app: istio-ingressgateway\n istio: ingressgateway\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: istio-ingressgateway-service-account\n containers:\n - name: ingressgateway\n image: \"docker.io/istio/proxyv2:1.0.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 80\n - containerPort: 443\n - containerPort: 31400\n - containerPort: 15011\n - containerPort: 8060\n - containerPort: 15030\n - containerPort: 15031\n args:\n - proxy\n - router\n - -v\n - \"2\"\n - --discoveryRefreshDelay\n - '1s' #discoveryRefreshDelay\n - --drainDuration\n - '45s' #drainDuration\n - --parentShutdownDuration\n - '1m0s' #parentShutdownDuration\n - --connectTimeout\n - '10s' #connectTimeout\n - --serviceCluster\n - istio-ingressgateway\n - --zipkinAddress\n - zipkin:9411\n - --statsdUdpAddress\n - istio-statsd-prom-bridge:9125\n - --proxyAdminPort\n - \"15000\"\n - --controlPlaneAuthPolicy\n - NONE\n - --discoveryAddress\n - istio-pilot.istio-system:8080\n resources:\n requests:\n cpu: 10m\n \n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n - name: ISTIO_META_POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n - name: ingressgateway-certs\n mountPath: \"/etc/istio/ingressgateway-certs\"\n readOnly: true\n - name: ingressgateway-ca-certs\n mountPath: \"/etc/istio/ingressgateway-ca-certs\"\n readOnly: true\n volumes:\n - name: istio-certs\n secret:\n secretName: istio.istio-ingressgateway-service-account\n optional: true\n - name: ingressgateway-certs\n secret:\n secretName: \"istio-ingressgateway-certs\"\n optional: true\n - name: ingressgateway-ca-certs\n secret:\n secretName: \"istio-ingressgateway-ca-certs\"\n optional: true\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n---\n\n---\n# Source: istio/charts/mixer/templates/deployment.yaml\n\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-policy\n namespace: istio-system\n labels:\n chart: mixer-1.0.0\n release: istio\n istio: mixer\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n app: policy\n istio: mixer\n istio-mixer-type: policy\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: istio-mixer-service-account\n volumes:\n - name: istio-certs\n secret:\n secretName: istio.istio-mixer-service-account\n optional: true\n - name: uds-socket\n emptyDir: {}\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n containers:\n - name: mixer\n image: \"docker.io/istio/mixer:1.0.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9093\n - containerPort: 42422\n args:\n - --address\n - unix:///sock/mixer.socket\n - --configStoreURL=k8s://\n - --configDefaultNamespace=istio-system\n - --trace_zipkin_url=http://zipkin:9411/api/v1/spans\n resources:\n requests:\n cpu: 10m\n \n volumeMounts:\n - name: uds-socket\n mountPath: /sock\n livenessProbe:\n httpGet:\n path: /version\n port: 9093\n initialDelaySeconds: 5\n periodSeconds: 5\n - name: istio-proxy\n image: \"docker.io/istio/proxyv2:1.0.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9091\n - containerPort: 15004\n args:\n - proxy\n - --serviceCluster\n - istio-policy\n - --templateFile\n - /etc/istio/proxy/envoy_policy.yaml.tmpl\n - --controlPlaneAuthPolicy\n - NONE\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n resources:\n requests:\n cpu: 10m\n \n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n - name: uds-socket\n mountPath: /sock\n\n---\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-telemetry\n namespace: istio-system\n labels:\n chart: mixer-1.0.0\n release: istio\n istio: mixer\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n app: telemetry\n istio: mixer\n istio-mixer-type: telemetry\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: istio-mixer-service-account\n volumes:\n - name: istio-certs\n secret:\n secretName: istio.istio-mixer-service-account\n optional: true\n - name: uds-socket\n emptyDir: {}\n containers:\n - name: mixer\n image: \"docker.io/istio/mixer:1.0.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9093\n - containerPort: 42422\n args:\n - --address\n - unix:///sock/mixer.socket\n - --configStoreURL=k8s://\n - --configDefaultNamespace=istio-system\n - --trace_zipkin_url=http://zipkin:9411/api/v1/spans\n resources:\n requests:\n cpu: 10m\n \n volumeMounts:\n - name: uds-socket\n mountPath: /sock\n livenessProbe:\n httpGet:\n path: /version\n port: 9093\n initialDelaySeconds: 5\n periodSeconds: 5\n - name: istio-proxy\n image: \"docker.io/istio/proxyv2:1.0.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 9091\n - containerPort: 15004\n args:\n - proxy\n - --serviceCluster\n - istio-telemetry\n - --templateFile\n - /etc/istio/proxy/envoy_telemetry.yaml.tmpl\n - --controlPlaneAuthPolicy\n - NONE\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n resources:\n requests:\n cpu: 10m\n \n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n - name: uds-socket\n mountPath: /sock\n\n--- \n\n---\n# Source: istio/charts/pilot/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-pilot\n namespace: istio-system\n # TODO: default template doesn't have this, which one is right ?\n labels:\n app: istio-pilot\n chart: pilot-1.0.0\n release: istio\n heritage: Tiller\n istio: pilot\n annotations:\n checksum/config-volume: f8da08b6b8c170dde721efd680270b2901e750d4aa186ebb6c22bef5b78a43f9\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n istio: pilot\n app: pilot\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: istio-pilot-service-account\n containers:\n - name: discovery\n image: \"docker.io/istio/pilot:1.0.0\"\n imagePullPolicy: IfNotPresent\n args:\n - \"discovery\"\n ports:\n - containerPort: 8080\n - containerPort: 15010\n readinessProbe:\n httpGet:\n path: /debug/endpointz\n port: 8080\n initialDelaySeconds: 30\n periodSeconds: 30\n timeoutSeconds: 5\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: PILOT_THROTTLE\n value: \"500\"\n - name: PILOT_CACHE_SQUASH\n value: \"5\"\n - name: PILOT_TRACE_SAMPLING\n value: \"100\"\n resources:\n requests:\n cpu: 500m\n memory: 2048Mi\n \n volumeMounts:\n - name: config-volume\n mountPath: /etc/istio/config\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n - name: istio-proxy\n image: \"docker.io/istio/proxyv2:1.0.0\"\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 15003\n - containerPort: 15005\n - containerPort: 15007\n - containerPort: 15011\n args:\n - proxy\n - --serviceCluster\n - istio-pilot\n - --templateFile\n - /etc/istio/proxy/envoy_pilot.yaml.tmpl\n - --controlPlaneAuthPolicy\n - NONE\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: INSTANCE_IP\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: status.podIP\n resources:\n requests:\n cpu: 10m\n \n volumeMounts:\n - name: istio-certs\n mountPath: /etc/certs\n readOnly: true\n volumes:\n - name: config-volume\n configMap:\n name: istio\n - name: istio-certs\n secret:\n secretName: istio.istio-pilot-service-account\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/prometheus/templates/deployment.yaml\n# TODO: the original template has service account, roles, etc\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: prometheus\n namespace: istio-system\n labels:\n app: prometheus\n chart: prometheus-0.1.0\n release: istio\n heritage: Tiller\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: prometheus\n template:\n metadata:\n labels:\n app: prometheus\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: prometheus\n containers:\n - name: prometheus\n image: \"docker.io/prom/prometheus:v2.3.1\"\n imagePullPolicy: IfNotPresent\n args:\n - '--storage.tsdb.retention=6h'\n - '--config.file=/etc/prometheus/prometheus.yml'\n ports:\n - containerPort: 9090\n name: http\n livenessProbe:\n httpGet:\n path: /-/healthy\n port: 9090\n readinessProbe:\n httpGet:\n path: /-/ready\n port: 9090\n resources:\n requests:\n cpu: 10m\n \n volumeMounts:\n - name: config-volume\n mountPath: /etc/prometheus\n volumes:\n - name: config-volume\n configMap:\n name: prometheus\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/security/templates/deployment.yaml\n# istio CA watching all namespaces\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-citadel\n namespace: istio-system\n labels:\n app: security\n chart: security-1.0.0\n release: istio\n heritage: Tiller\n istio: citadel\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n istio: citadel\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: istio-citadel-service-account\n containers:\n - name: citadel\n image: \"docker.io/istio/citadel:1.0.0\"\n imagePullPolicy: IfNotPresent\n args:\n - --append-dns-names=true\n - --grpc-port=8060\n - --grpc-hostname=citadel\n - --citadel-storage-namespace=istio-system\n - --self-signed-ca=true\n resources:\n requests:\n cpu: 10m\n \n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/deployment.yaml\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n name: istio-sidecar-injector\n namespace: istio-system\n labels:\n app: sidecarInjectorWebhook\n chart: sidecarInjectorWebhook-1.0.0\n release: istio\n heritage: Tiller\n istio: sidecar-injector\nspec:\n replicas: 1\n template:\n metadata:\n labels:\n istio: sidecar-injector\n annotations:\n sidecar.istio.io/inject: \"false\"\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n serviceAccountName: istio-sidecar-injector-service-account\n containers:\n - name: sidecar-injector-webhook\n image: \"docker.io/istio/sidecar_injector:1.0.0\"\n imagePullPolicy: IfNotPresent\n args:\n - --caCertFile=/etc/istio/certs/root-cert.pem\n - --tlsCertFile=/etc/istio/certs/cert-chain.pem\n - --tlsKeyFile=/etc/istio/certs/key.pem\n - --injectConfig=/etc/istio/inject/config\n - --meshConfig=/etc/istio/config/mesh\n - --healthCheckInterval=2s\n - --healthCheckFile=/health\n volumeMounts:\n - name: config-volume\n mountPath: /etc/istio/config\n readOnly: true\n - name: certs\n mountPath: /etc/istio/certs\n readOnly: true\n - name: inject-config\n mountPath: /etc/istio/inject\n readOnly: true\n livenessProbe:\n exec:\n command:\n - /usr/local/bin/sidecar-injector\n - probe\n - --probe-path=/health\n - --interval=4s\n initialDelaySeconds: 4\n periodSeconds: 4\n readinessProbe:\n exec:\n command:\n - /usr/local/bin/sidecar-injector\n - probe\n - --probe-path=/health\n - --interval=4s\n initialDelaySeconds: 4\n periodSeconds: 4\n resources:\n requests:\n cpu: 10m\n \n volumes:\n - name: config-volume\n configMap:\n name: istio\n - name: certs\n secret:\n secretName: istio.istio-sidecar-injector-service-account\n - name: inject-config\n configMap:\n name: istio-sidecar-injector\n items:\n - key: config\n path: config\n affinity: \n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - ppc64le\n - s390x\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - amd64\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - ppc64le\n - weight: 2\n preference:\n matchExpressions:\n - key: beta.kubernetes.io/arch\n operator: In\n values:\n - s390x\n\n---\n# Source: istio/charts/pilot/templates/gateway.yaml\napiVersion: networking.istio.io/v1alpha3\nkind: Gateway\nmetadata:\n name: istio-autogenerated-k8s-ingress\n namespace: istio-system\nspec:\n selector:\n istio: ingress\n servers:\n - port:\n number: 80\n protocol: HTTP2\n name: http\n hosts:\n - \"*\"\n\n---\n\n---\n# Source: istio/charts/gateways/templates/autoscale.yaml\n\napiVersion: autoscaling/v2beta1\nkind: HorizontalPodAutoscaler\nmetadata:\n name: istio-egressgateway\n namespace: istio-system\nspec:\n maxReplicas: 5\n minReplicas: 1\n scaleTargetRef:\n apiVersion: apps/v1beta1\n kind: Deployment\n name: istio-egressgateway\n metrics:\n - type: Resource\n resource:\n name: cpu\n targetAverageUtilization: 60\n---\napiVersion: autoscaling/v2beta1\nkind: HorizontalPodAutoscaler\nmetadata:\n name: istio-ingressgateway\n namespace: istio-system\nspec:\n maxReplicas: 5\n minReplicas: 1\n scaleTargetRef:\n apiVersion: apps/v1beta1\n kind: Deployment\n name: istio-ingressgateway\n metrics:\n - type: Resource\n resource:\n name: cpu\n targetAverageUtilization: 60\n---\n\n---\n# Source: istio/charts/mixer/templates/autoscale.yaml\n\napiVersion: autoscaling/v2beta1\nkind: HorizontalPodAutoscaler\nmetadata:\n name: istio-policy\n namespace: istio-system\nspec:\n maxReplicas: 5\n minReplicas: 1\n scaleTargetRef:\n apiVersion: apps/v1beta1\n kind: Deployment\n name: istio-policy\n metrics:\n - type: Resource\n resource:\n name: cpu\n targetAverageUtilization: 80\n---\napiVersion: autoscaling/v2beta1\nkind: HorizontalPodAutoscaler\nmetadata:\n name: istio-telemetry\n namespace: istio-system\nspec:\n maxReplicas: 5\n minReplicas: 1\n scaleTargetRef:\n apiVersion: apps/v1beta1\n kind: Deployment\n name: istio-telemetry\n metrics:\n - type: Resource\n resource:\n name: cpu\n targetAverageUtilization: 80\n---\n\n---\n# Source: istio/charts/pilot/templates/autoscale.yaml\n\napiVersion: autoscaling/v2beta1\nkind: HorizontalPodAutoscaler\nmetadata:\n name: istio-pilot\nspec:\n maxReplicas: 1\n minReplicas: 1\n scaleTargetRef:\n apiVersion: apps/v1beta1\n kind: Deployment\n name: istio-pilot\n metrics:\n - type: Resource\n resource:\n name: cpu\n targetAverageUtilization: 55\n---\n\n---\n# Source: istio/charts/sidecarInjectorWebhook/templates/mutatingwebhook.yaml\napiVersion: admissionregistration.k8s.io/v1beta1\nkind: MutatingWebhookConfiguration\nmetadata:\n name: istio-sidecar-injector\n namespace: istio-system\n labels:\n app: istio-sidecar-injector\n chart: sidecarInjectorWebhook-1.0.0\n release: istio\n heritage: Tiller\nwebhooks:\n - name: sidecar-injector.istio.io\n clientConfig:\n service:\n name: istio-sidecar-injector\n namespace: istio-system\n path: \"/inject\"\n caBundle: \"\"\n rules:\n - operations: [ \"CREATE\" ]\n apiGroups: [\"\"]\n apiVersions: [\"v1\"]\n resources: [\"pods\"]\n failurePolicy: Fail\n namespaceSelector:\n matchLabels:\n istio-injection: enabled\n\n\n---\n# Source: istio/charts/galley/templates/validatingwehookconfiguration.yaml.tpl\n\n\n---\n# Source: istio/charts/pilot/templates/meshexpansion.yaml\n\n\n---\n# Source: istio/charts/security/templates/create-custom-resources-job.yaml\n\n\n---\n# Source: istio/charts/security/templates/enable-mesh-mtls.yaml\n\n\n---\n# Source: istio/charts/security/templates/meshexpansion.yaml\n\n\n---\n\n---\n# Source: istio/charts/telemetry-gateway/templates/gateway.yaml\n\n\n---\n# Source: istio/templates/install-custom-resources.sh.tpl\n\n\n---\n# Source: istio/charts/mixer/templates/config.yaml\napiVersion: \"config.istio.io/v1alpha2\"\nkind: attributemanifest\nmetadata:\n name: istioproxy\n namespace: istio-system\nspec:\n attributes:\n origin.ip:\n valueType: IP_ADDRESS\n origin.uid:\n valueType: STRING\n origin.user:\n valueType: STRING\n request.headers:\n valueType: STRING_MAP\n request.id:\n valueType: STRING\n request.host:\n valueType: STRING\n request.method:\n valueType: STRING\n request.path:\n valueType: STRING\n request.reason:\n valueType: STRING\n request.referer:\n valueType: STRING\n request.scheme:\n valueType: STRING\n request.total_size:\n valueType: INT64\n request.size:\n valueType: INT64\n request.time:\n valueType: TIMESTAMP\n request.useragent:\n valueType: STRING\n response.code:\n valueType: INT64\n response.duration:\n valueType: DURATION\n response.headers:\n valueType: STRING_MAP\n response.total_size:\n valueType: INT64\n response.size:\n valueType: INT64\n response.time:\n valueType: TIMESTAMP\n source.uid:\n valueType: STRING\n source.user: # DEPRECATED\n valueType: STRING\n source.principal:\n valueType: STRING\n destination.uid:\n valueType: STRING\n destination.principal:\n valueType: STRING\n destination.port:\n valueType: INT64\n connection.event:\n valueType: STRING\n connection.id:\n valueType: STRING\n connection.received.bytes:\n valueType: INT64\n connection.received.bytes_total:\n valueType: INT64\n connection.sent.bytes:\n valueType: INT64\n connection.sent.bytes_total:\n valueType: INT64\n connection.duration:\n valueType: DURATION\n connection.mtls:\n valueType: BOOL\n context.protocol:\n valueType: STRING\n context.timestamp:\n valueType: TIMESTAMP\n context.time:\n valueType: TIMESTAMP\n # Deprecated, kept for compatibility\n context.reporter.local:\n valueType: BOOL\n context.reporter.kind:\n valueType: STRING\n context.reporter.uid:\n valueType: STRING\n api.service:\n valueType: STRING\n api.version:\n valueType: STRING\n api.operation:\n valueType: STRING\n api.protocol:\n valueType: STRING\n request.auth.principal:\n valueType: STRING\n request.auth.audiences:\n valueType: STRING\n request.auth.presenter:\n valueType: STRING\n request.auth.claims:\n valueType: STRING_MAP\n request.auth.raw_claims:\n valueType: STRING\n request.api_key:\n valueType: STRING\n\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: attributemanifest\nmetadata:\n name: kubernetes\n namespace: istio-system\nspec:\n attributes:\n source.ip:\n valueType: IP_ADDRESS\n source.labels:\n valueType: STRING_MAP\n source.metadata:\n valueType: STRING_MAP\n source.name:\n valueType: STRING\n source.namespace:\n valueType: STRING\n source.owner:\n valueType: STRING\n source.service: # DEPRECATED\n valueType: STRING\n source.serviceAccount:\n valueType: STRING\n source.services:\n valueType: STRING\n source.workload.uid:\n valueType: STRING\n source.workload.name:\n valueType: STRING\n source.workload.namespace:\n valueType: STRING\n destination.ip:\n valueType: IP_ADDRESS\n destination.labels:\n valueType: STRING_MAP\n destination.metadata:\n valueType: STRING_MAP\n destination.owner:\n valueType: STRING\n destination.name:\n valueType: STRING\n destination.container.name:\n valueType: STRING\n destination.namespace:\n valueType: STRING\n destination.service: # DEPRECATED\n valueType: STRING\n destination.service.uid:\n valueType: STRING\n destination.service.name:\n valueType: STRING\n destination.service.namespace:\n valueType: STRING\n destination.service.host:\n valueType: STRING\n destination.serviceAccount:\n valueType: STRING\n destination.workload.uid:\n valueType: STRING\n destination.workload.name:\n valueType: STRING\n destination.workload.namespace:\n valueType: STRING\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: stdio\nmetadata:\n name: handler\n namespace: istio-system\nspec:\n outputAsJson: true\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: logentry\nmetadata:\n name: accesslog\n namespace: istio-system\nspec:\n severity: '\"Info\"'\n timestamp: request.time\n variables:\n sourceIp: source.ip | ip(\"0.0.0.0\")\n sourceApp: source.labels[\"app\"] | \"\"\n sourcePrincipal: source.principal | \"\"\n sourceName: source.name | \"\"\n sourceWorkload: source.workload.name | \"\"\n sourceNamespace: source.namespace | \"\"\n sourceOwner: source.owner | \"\"\n destinationApp: destination.labels[\"app\"] | \"\"\n destinationIp: destination.ip | ip(\"0.0.0.0\")\n destinationServiceHost: destination.service.host | \"\"\n destinationWorkload: destination.workload.name | \"\"\n destinationName: destination.name | \"\"\n destinationNamespace: destination.namespace | \"\"\n destinationOwner: destination.owner | \"\"\n destinationPrincipal: destination.principal | \"\"\n apiClaims: request.auth.raw_claims | \"\"\n apiKey: request.api_key | request.headers[\"x-api-key\"] | \"\"\n protocol: request.scheme | context.protocol | \"http\"\n method: request.method | \"\"\n url: request.path | \"\"\n responseCode: response.code | 0\n responseSize: response.size | 0\n requestSize: request.size | 0\n requestId: request.headers[\"x-request-id\"] | \"\"\n clientTraceId: request.headers[\"x-client-trace-id\"] | \"\"\n latency: response.duration | \"0ms\"\n connection_security_policy: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"unknown\", conditional(connection.mtls | false, \"mutual_tls\", \"none\"))\n userAgent: request.useragent | \"\"\n responseTimestamp: response.time\n receivedBytes: request.total_size | 0\n sentBytes: response.total_size | 0\n referer: request.referer | \"\"\n httpAuthority: request.headers[\":authority\"] | request.host | \"\"\n xForwardedFor: request.headers[\"x-forwarded-for\"] | \"0.0.0.0\"\n reporter: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"source\", \"destination\")\n monitored_resource_type: '\"global\"'\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: logentry\nmetadata:\n name: tcpaccesslog\n namespace: istio-system\nspec:\n severity: '\"Info\"'\n timestamp: context.time | timestamp(\"2017-01-01T00:00:00Z\")\n variables:\n connectionEvent: connection.event | \"\"\n sourceIp: source.ip | ip(\"0.0.0.0\")\n sourceApp: source.labels[\"app\"] | \"\"\n sourcePrincipal: source.principal | \"\"\n sourceName: source.name | \"\"\n sourceWorkload: source.workload.name | \"\"\n sourceNamespace: source.namespace | \"\"\n sourceOwner: source.owner | \"\"\n destinationApp: destination.labels[\"app\"] | \"\"\n destinationIp: destination.ip | ip(\"0.0.0.0\")\n destinationServiceHost: destination.service.host | \"\"\n destinationWorkload: destination.workload.name | \"\"\n destinationName: destination.name | \"\"\n destinationNamespace: destination.namespace | \"\"\n destinationOwner: destination.owner | \"\"\n destinationPrincipal: destination.principal | \"\"\n protocol: context.protocol | \"tcp\"\n connectionDuration: connection.duration | \"0ms\"\n connection_security_policy: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"unknown\", conditional(connection.mtls | false, \"mutual_tls\", \"none\"))\n receivedBytes: connection.received.bytes | 0\n sentBytes: connection.sent.bytes | 0\n totalReceivedBytes: connection.received.bytes_total | 0\n totalSentBytes: connection.sent.bytes_total | 0\n reporter: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"source\", \"destination\")\n monitored_resource_type: '\"global\"'\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: rule\nmetadata:\n name: stdio\n namespace: istio-system\nspec:\n match: context.protocol == \"http\" || context.protocol == \"grpc\"\n actions:\n - handler: handler.stdio\n instances:\n - accesslog.logentry\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: rule\nmetadata:\n name: stdiotcp\n namespace: istio-system\nspec:\n match: context.protocol == \"tcp\"\n actions:\n - handler: handler.stdio\n instances:\n - tcpaccesslog.logentry\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: metric\nmetadata:\n name: requestcount\n namespace: istio-system\nspec:\n value: \"1\"\n dimensions:\n reporter: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"source\", \"destination\")\n source_workload: source.workload.name | \"unknown\"\n source_workload_namespace: source.workload.namespace | \"unknown\"\n source_principal: source.principal | \"unknown\"\n source_app: source.labels[\"app\"] | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_workload: destination.workload.name | \"unknown\"\n destination_workload_namespace: destination.workload.namespace | \"unknown\"\n destination_principal: destination.principal | \"unknown\"\n destination_app: destination.labels[\"app\"] | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n destination_service: destination.service.host | \"unknown\"\n destination_service_name: destination.service.name | \"unknown\"\n destination_service_namespace: destination.service.namespace | \"unknown\"\n request_protocol: api.protocol | context.protocol | \"unknown\"\n response_code: response.code | 200\n connection_security_policy: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"unknown\", conditional(connection.mtls | false, \"mutual_tls\", \"none\"))\n monitored_resource_type: '\"UNSPECIFIED\"'\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: metric\nmetadata:\n name: requestduration\n namespace: istio-system\nspec:\n value: response.duration | \"0ms\"\n dimensions:\n reporter: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"source\", \"destination\")\n source_workload: source.workload.name | \"unknown\"\n source_workload_namespace: source.workload.namespace | \"unknown\"\n source_principal: source.principal | \"unknown\"\n source_app: source.labels[\"app\"] | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_workload: destination.workload.name | \"unknown\"\n destination_workload_namespace: destination.workload.namespace | \"unknown\"\n destination_principal: destination.principal | \"unknown\"\n destination_app: destination.labels[\"app\"] | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n destination_service: destination.service.host | \"unknown\"\n destination_service_name: destination.service.name | \"unknown\"\n destination_service_namespace: destination.service.namespace | \"unknown\"\n request_protocol: api.protocol | context.protocol | \"unknown\"\n response_code: response.code | 200\n connection_security_policy: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"unknown\", conditional(connection.mtls | false, \"mutual_tls\", \"none\"))\n monitored_resource_type: '\"UNSPECIFIED\"'\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: metric\nmetadata:\n name: requestsize\n namespace: istio-system\nspec:\n value: request.size | 0\n dimensions:\n reporter: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"source\", \"destination\")\n source_workload: source.workload.name | \"unknown\"\n source_workload_namespace: source.workload.namespace | \"unknown\"\n source_principal: source.principal | \"unknown\"\n source_app: source.labels[\"app\"] | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_workload: destination.workload.name | \"unknown\"\n destination_workload_namespace: destination.workload.namespace | \"unknown\"\n destination_principal: destination.principal | \"unknown\"\n destination_app: destination.labels[\"app\"] | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n destination_service: destination.service.host | \"unknown\"\n destination_service_name: destination.service.name | \"unknown\"\n destination_service_namespace: destination.service.namespace | \"unknown\"\n request_protocol: api.protocol | context.protocol | \"unknown\"\n response_code: response.code | 200\n connection_security_policy: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"unknown\", conditional(connection.mtls | false, \"mutual_tls\", \"none\"))\n monitored_resource_type: '\"UNSPECIFIED\"'\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: metric\nmetadata:\n name: responsesize\n namespace: istio-system\nspec:\n value: response.size | 0\n dimensions:\n reporter: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"source\", \"destination\")\n source_workload: source.workload.name | \"unknown\"\n source_workload_namespace: source.workload.namespace | \"unknown\"\n source_principal: source.principal | \"unknown\"\n source_app: source.labels[\"app\"] | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_workload: destination.workload.name | \"unknown\"\n destination_workload_namespace: destination.workload.namespace | \"unknown\"\n destination_principal: destination.principal | \"unknown\"\n destination_app: destination.labels[\"app\"] | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n destination_service: destination.service.host | \"unknown\"\n destination_service_name: destination.service.name | \"unknown\"\n destination_service_namespace: destination.service.namespace | \"unknown\"\n request_protocol: api.protocol | context.protocol | \"unknown\"\n response_code: response.code | 200\n connection_security_policy: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"unknown\", conditional(connection.mtls | false, \"mutual_tls\", \"none\"))\n monitored_resource_type: '\"UNSPECIFIED\"'\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: metric\nmetadata:\n name: tcpbytesent\n namespace: istio-system\nspec:\n value: connection.sent.bytes | 0\n dimensions:\n reporter: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"source\", \"destination\")\n source_workload: source.workload.name | \"unknown\"\n source_workload_namespace: source.workload.namespace | \"unknown\"\n source_principal: source.principal | \"unknown\"\n source_app: source.labels[\"app\"] | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_workload: destination.workload.name | \"unknown\"\n destination_workload_namespace: destination.workload.namespace | \"unknown\"\n destination_principal: destination.principal | \"unknown\"\n destination_app: destination.labels[\"app\"] | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n destination_service: destination.service.name | \"unknown\"\n destination_service_name: destination.service.name | \"unknown\"\n destination_service_namespace: destination.service.namespace | \"unknown\"\n connection_security_policy: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"unknown\", conditional(connection.mtls | false, \"mutual_tls\", \"none\"))\n monitored_resource_type: '\"UNSPECIFIED\"'\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: metric\nmetadata:\n name: tcpbytereceived\n namespace: istio-system\nspec:\n value: connection.received.bytes | 0\n dimensions:\n reporter: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"source\", \"destination\")\n source_workload: source.workload.name | \"unknown\"\n source_workload_namespace: source.workload.namespace | \"unknown\"\n source_principal: source.principal | \"unknown\"\n source_app: source.labels[\"app\"] | \"unknown\"\n source_version: source.labels[\"version\"] | \"unknown\"\n destination_workload: destination.workload.name | \"unknown\"\n destination_workload_namespace: destination.workload.namespace | \"unknown\"\n destination_principal: destination.principal | \"unknown\"\n destination_app: destination.labels[\"app\"] | \"unknown\"\n destination_version: destination.labels[\"version\"] | \"unknown\"\n destination_service: destination.service.name | \"unknown\"\n destination_service_name: destination.service.name | \"unknown\"\n destination_service_namespace: destination.service.namespace | \"unknown\"\n connection_security_policy: conditional((context.reporter.kind | \"inbound\") == \"outbound\", \"unknown\", conditional(connection.mtls | false, \"mutual_tls\", \"none\"))\n monitored_resource_type: '\"UNSPECIFIED\"'\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: prometheus\nmetadata:\n name: handler\n namespace: istio-system\nspec:\n metrics:\n - name: requests_total\n instance_name: requestcount.metric.istio-system\n kind: COUNTER\n label_names:\n - reporter\n - source_app\n - source_principal\n - source_workload\n - source_workload_namespace\n - source_version\n - destination_app\n - destination_principal\n - destination_workload\n - destination_workload_namespace\n - destination_version\n - destination_service\n - destination_service_name\n - destination_service_namespace\n - request_protocol\n - response_code\n - connection_security_policy\n - name: request_duration_seconds\n instance_name: requestduration.metric.istio-system\n kind: DISTRIBUTION\n label_names:\n - reporter\n - source_app\n - source_principal\n - source_workload\n - source_workload_namespace\n - source_version\n - destination_app\n - destination_principal\n - destination_workload\n - destination_workload_namespace\n - destination_version\n - destination_service\n - destination_service_name\n - destination_service_namespace\n - request_protocol\n - response_code\n - connection_security_policy\n buckets:\n explicit_buckets:\n bounds: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]\n - name: request_bytes\n instance_name: requestsize.metric.istio-system\n kind: DISTRIBUTION\n label_names:\n - reporter\n - source_app\n - source_principal\n - source_workload\n - source_workload_namespace\n - source_version\n - destination_app\n - destination_principal\n - destination_workload\n - destination_workload_namespace\n - destination_version\n - destination_service\n - destination_service_name\n - destination_service_namespace\n - request_protocol\n - response_code\n - connection_security_policy\n buckets:\n exponentialBuckets:\n numFiniteBuckets: 8\n scale: 1\n growthFactor: 10\n - name: response_bytes\n instance_name: responsesize.metric.istio-system\n kind: DISTRIBUTION\n label_names:\n - reporter\n - source_app\n - source_principal\n - source_workload\n - source_workload_namespace\n - source_version\n - destination_app\n - destination_principal\n - destination_workload\n - destination_workload_namespace\n - destination_version\n - destination_service\n - destination_service_name\n - destination_service_namespace\n - request_protocol\n - response_code\n - connection_security_policy\n buckets:\n exponentialBuckets:\n numFiniteBuckets: 8\n scale: 1\n growthFactor: 10\n - name: tcp_sent_bytes_total\n instance_name: tcpbytesent.metric.istio-system\n kind: COUNTER\n label_names:\n - reporter\n - source_app\n - source_principal\n - source_workload\n - source_workload_namespace\n - source_version\n - destination_app\n - destination_principal\n - destination_workload\n - destination_workload_namespace\n - destination_version\n - destination_service\n - destination_service_name\n - destination_service_namespace\n - connection_security_policy\n - name: tcp_received_bytes_total\n instance_name: tcpbytereceived.metric.istio-system\n kind: COUNTER\n label_names:\n - reporter\n - source_app\n - source_principal\n - source_workload\n - source_workload_namespace\n - source_version\n - destination_app\n - destination_principal\n - destination_workload\n - destination_workload_namespace\n - destination_version\n - destination_service\n - destination_service_name\n - destination_service_namespace\n - connection_security_policy\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: rule\nmetadata:\n name: promhttp\n namespace: istio-system\nspec:\n match: context.protocol == \"http\" || context.protocol == \"grpc\"\n actions:\n - handler: handler.prometheus\n instances:\n - requestcount.metric\n - requestduration.metric\n - requestsize.metric\n - responsesize.metric\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: rule\nmetadata:\n name: promtcp\n namespace: istio-system\nspec:\n match: context.protocol == \"tcp\"\n actions:\n - handler: handler.prometheus\n instances:\n - tcpbytesent.metric\n - tcpbytereceived.metric\n---\n\napiVersion: \"config.istio.io/v1alpha2\"\nkind: kubernetesenv\nmetadata:\n name: handler\n namespace: istio-system\nspec:\n # when running from mixer root, use the following config after adding a\n # symbolic link to a kubernetes config file via:\n #\n # $ ln -s ~/.kube/config mixer/adapter/kubernetes/kubeconfig\n #\n # kubeconfig_path: \"mixer/adapter/kubernetes/kubeconfig\"\n\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: rule\nmetadata:\n name: kubeattrgenrulerule\n namespace: istio-system\nspec:\n actions:\n - handler: handler.kubernetesenv\n instances:\n - attributes.kubernetes\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: rule\nmetadata:\n name: tcpkubeattrgenrulerule\n namespace: istio-system\nspec:\n match: context.protocol == \"tcp\"\n actions:\n - handler: handler.kubernetesenv\n instances:\n - attributes.kubernetes\n---\napiVersion: \"config.istio.io/v1alpha2\"\nkind: kubernetes\nmetadata:\n name: attributes\n namespace: istio-system\nspec:\n # Pass the required attribute data to the adapter\n source_uid: source.uid | \"\"\n source_ip: source.ip | ip(\"0.0.0.0\") # default to unspecified ip addr\n destination_uid: destination.uid | \"\"\n destination_port: destination.port | 0\n attribute_bindings:\n # Fill the new attributes from the adapter produced output.\n # $out refers to an instance of OutputTemplate message\n source.ip: $out.source_pod_ip | ip(\"0.0.0.0\")\n source.uid: $out.source_pod_uid | \"unknown\"\n source.labels: $out.source_labels | emptyStringMap()\n source.name: $out.source_pod_name | \"unknown\"\n source.namespace: $out.source_namespace | \"default\"\n source.owner: $out.source_owner | \"unknown\"\n source.serviceAccount: $out.source_service_account_name | \"unknown\"\n source.workload.uid: $out.source_workload_uid | \"unknown\"\n source.workload.name: $out.source_workload_name | \"unknown\"\n source.workload.namespace: $out.source_workload_namespace | \"unknown\"\n destination.ip: $out.destination_pod_ip | ip(\"0.0.0.0\")\n destination.uid: $out.destination_pod_uid | \"unknown\"\n destination.labels: $out.destination_labels | emptyStringMap()\n destination.name: $out.destination_pod_name | \"unknown\"\n destination.container.name: $out.destination_container_name | \"unknown\"\n destination.namespace: $out.destination_namespace | \"default\"\n destination.owner: $out.destination_owner | \"unknown\"\n destination.serviceAccount: $out.destination_service_account_name | \"unknown\"\n destination.workload.uid: $out.destination_workload_uid | \"unknown\"\n destination.workload.name: $out.destination_workload_name | \"unknown\"\n destination.workload.namespace: $out.destination_workload_namespace | \"unknown\"\n\n---\n# Configuration needed by Mixer.\n# Mixer cluster is delivered via CDS\n# Specify mixer cluster settings\napiVersion: networking.istio.io/v1alpha3\nkind: DestinationRule\nmetadata:\n name: istio-policy\n namespace: istio-system\nspec:\n host: istio-policy.istio-system.svc.cluster.local\n trafficPolicy:\n connectionPool:\n http:\n http2MaxRequests: 10000\n maxRequestsPerConnection: 10000\n---\napiVersion: networking.istio.io/v1alpha3\nkind: DestinationRule\nmetadata:\n name: istio-telemetry\n namespace: istio-system\nspec:\n host: istio-telemetry.istio-system.svc.cluster.local\n trafficPolicy:\n connectionPool:\n http:\n http2MaxRequests: 10000\n maxRequestsPerConnection: 10000\n---\n\n" }, { "path": "examples/90_Kubernetes/minikube/README.md", "content": "# Development with Minikube\n\n## Install `minikube`, `kubectl`, and `helm`\n\nThis only need to be done one time, or periodically if you wish to upgrade.\n\n```\nsudo apt update && sudo apt install -y --no-install-recommends socat\n\ncurl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/\n\ncurl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin\n\ncurl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh\nchmod 700 get_helm.sh\n./get_helm.sh\n```\n\nAdd the `coreos/prometheus-operator` repo:\n```\nhelm repo add coreos https://s3-eu-west-1.amazonaws.com/coreos-charts/stable/\n```\n\n## Launch a Kubernetes Cluster\n\n```\n./bootstrap.sh\n```\n\nCheck configurations:\n```\nkubectl get all\nkubectl get all --all-namespaces\nkubectl get nodes -o=custom-columns=NAME:.metadata.name,GPUs:.status.capacity.'nvidia\\.com/gpu'\n# last command should report the number of GPUs on your system\n# this make take some time - coffee?!\n```\n\n" }, { "path": "examples/90_Kubernetes/minikube/bootstrap.sh", "content": "#!/bin/bash\n\nmkdir -p $HOME/.kube\ntouch $HOME/.kube/config\n\nexport MINIKUBE_HOME=$HOME\nexport CHANGE_MINIKUBE_NONE_USER=true\nexport KUBECONFIG=$HOME/.kube/config\n\nversion=v1.10\n\nsudo minikube start \\\n --feature-gates=DevicePlugins=true \\\n --vm-driver=none \\\n --kubernetes-version=${version}.0 \\\n --bootstrapper=kubeadm \\\n --extra-config=kubelet.authentication-token-webhook=true \\\n --extra-config=kubelet.authorization-mode=Webhook \\\n --extra-config=scheduler.address=0.0.0.0 \\\n --extra-config=controller-manager.address=0.0.0.0 \\\n --extra-config=controller-manager.cluster-signing-cert-file=\"/var/lib/localkube/certs/ca.crt\" \\\n --extra-config=controller-manager.cluster-signing-key-file=\"/var/lib/localkube/certs/ca.key\" \\\n --extra-config=apiserver.admission-control=\"NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota\" \n\nif [ ! -e $HOME/.kube ]; then\n sudo mv /root/.kube $HOME/.kube > /dev/null 2>&1 ||: # this will write over any previous configuration\n sudo chown -R $USER $HOME/.kube > /dev/null 2>&1 ||:\n sudo chgrp -R $USER $HOME/.kube > /dev/null 2>&1 ||:\nfi\n\nif [ ! -e $HOME/.minikube ]; then\n sudo mv /root/.minikube $HOME/.minikube # > /dev/null 2>&1 ||: this will write over any previous configuration\n sudo chown -R $USER $HOME/.minikube > /dev/null 2>&1 ||:\n sudo chgrp -R $USER $HOME/.minikube > /dev/null 2>&1 ||:\nfi\n\nkubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/${version}/nvidia-device-plugin.yml\n\n# dns fix for dgx-stations using ubuntu's network manager\nkubectl apply -f https://raw.githubusercontent.com/ryanolson/k8s-upstream-dns/master/dns.yml\n" }, { "path": "examples/90_Kubernetes/prometheus/bootstrap.sh", "content": "#!/bin/bash\n\nkubectl create -f service-account.yml\n\nhelm init --wait --service-account tiller\n\nhelm repo add coreos https://s3-eu-west-1.amazonaws.com/coreos-charts/stable/\n\nhelm install coreos/prometheus-operator \\\n --name prometheus-operator \\\n --namespace monitoring\n\nhelm install coreos/kube-prometheus \\\n --name kube-prometheus \\\n --namespace monitoring \\\n -f custom-settings.yml\n\nkubectl apply -f yais-metrics.yml\n" }, { "path": "examples/90_Kubernetes/prometheus/custom-settings.yml", "content": "global:\n rbacEnable: true\n\n#prometheus:\n# service:\n# type: NodePort\n\ngrafana:\n# image:\n# tag: 5.2.1\n service:\n type: NodePort\n dataSource:\n yais-datasource.json: |+\n {\n \"access\": \"proxy\",\n \"basicAuth\": false,\n \"name\": \"yais\",\n \"type\": \"prometheus\",\n \"url\": \"http://yais-metrics.default:9090\"\n }\n serverDashboardFiles:\n yais-dashboard.json: |+\n {\n \"dashboard\": \n {\n \"__inputs\": [\n {\n \"name\": \"DS_YAIS\",\n \"label\": \"yais\",\n \"description\": \"\",\n \"type\": \"datasource\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\"\n }\n ],\n \"__requires\": [\n {\n \"type\": \"grafana\",\n \"id\": \"grafana\",\n \"name\": \"Grafana\",\n \"version\": \"5.0.0\"\n },\n {\n \"type\": \"panel\",\n \"id\": \"graph\",\n \"name\": \"Graph\",\n \"version\": \"5.0.0\"\n },\n {\n \"type\": \"panel\",\n \"id\": \"heatmap\",\n \"name\": \"Heatmap\",\n \"version\": \"5.0.0\"\n },\n {\n \"type\": \"datasource\",\n \"id\": \"prometheus\",\n \"name\": \"Prometheus\",\n \"version\": \"5.0.0\"\n }\n ],\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"gnetId\": null,\n \"graphTooltip\": 0,\n \"id\": null,\n \"links\": [],\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": true,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_YAIS}\",\n \"fill\": 1,\n \"gridPos\": {\n \"h\": 9,\n \"w\": 12,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 8,\n \"legend\": {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\": false,\n \"show\": false,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": true,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"irate(yais_inference_load_ratio_count[5s])\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n },\n {\n \"expr\": \"\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"B\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Inference Rate (FPS)\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_YAIS}\",\n \"fill\": 1,\n \"gridPos\": {\n \"h\": 9,\n \"w\": 12,\n \"x\": 12,\n \"y\": 0\n },\n \"id\": 4,\n \"legend\": {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\": false,\n \"show\": false,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": true,\n \"steppedLine\": true,\n \"targets\": [\n {\n \"expr\": \"yais_gpus_power_usage\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"GPU Power (Watts)\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_YAIS}\",\n \"fill\": 1,\n \"gridPos\": {\n \"h\": 9,\n \"w\": 12,\n \"x\": 0,\n \"y\": 9\n },\n \"id\": 2,\n \"legend\": {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\": false,\n \"show\": false,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"yais_executor_queue_depth\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Queue Depth\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"cards\": {\n \"cardPadding\": null,\n \"cardRound\": null\n },\n \"color\": {\n \"cardColor\": \"#b4ff00\",\n \"colorScale\": \"sqrt\",\n \"colorScheme\": \"interpolateOranges\",\n \"exponent\": 0.5,\n \"mode\": \"spectrum\"\n },\n \"dataFormat\": \"timeseries\",\n \"datasource\": \"${DS_YAIS}\",\n \"gridPos\": {\n \"h\": 9,\n \"w\": 12,\n \"x\": 12,\n \"y\": 9\n },\n \"heatmap\": {},\n \"highlightCards\": true,\n \"id\": 10,\n \"interval\": \"\",\n \"legend\": {\n \"show\": false\n },\n \"links\": [],\n \"targets\": [\n {\n \"expr\": \"irate(yais_inference_load_ratio_bucket[15s])\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"\",\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Load Ratio (Ideal 0-1)\",\n \"tooltip\": {\n \"show\": true,\n \"showHistogram\": false\n },\n \"type\": \"heatmap\",\n \"xAxis\": {\n \"show\": true\n },\n \"xBucketNumber\": null,\n \"xBucketSize\": null,\n \"yAxis\": {\n \"decimals\": null,\n \"format\": \"short\",\n \"logBase\": 10,\n \"max\": null,\n \"min\": \"0\",\n \"show\": true,\n \"splitFactor\": 2\n },\n \"yBucketNumber\": null,\n \"yBucketSize\": null\n }\n ],\n \"refresh\": \"5s\",\n \"schemaVersion\": 16,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\": \"now-30m\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"\",\n \"title\": \"YAIS\",\n \"uid\": \"3WLjQkdmk\",\n \"version\": 1\n },\n \"inputs\": [\n {\n \"name\": \"DS_YAIS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"yais\"\n }\n ],\n \"overwrite\": true\n }\n" }, { "path": "examples/90_Kubernetes/prometheus/service-account.yml", "content": "# Create a service account for Helm and grant the cluster admin role.\n# It is assumed that helm should be installed with this service account\n# (tiller).\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: tiller\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: tiller\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n- kind: ServiceAccount\n name: tiller\n namespace: kube-system\n" }, { "path": "examples/90_Kubernetes/prometheus/yais-dashboard.json", "content": "{\n \"__inputs\": [\n {\n \"name\": \"DS_YAIS\",\n \"label\": \"yais\",\n \"description\": \"\",\n \"type\": \"datasource\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\"\n }\n ],\n \"__requires\": [\n {\n \"type\": \"grafana\",\n \"id\": \"grafana\",\n \"name\": \"Grafana\",\n \"version\": \"5.2.1\"\n },\n {\n \"type\": \"panel\",\n \"id\": \"graph\",\n \"name\": \"Graph\",\n \"version\": \"5.0.0\"\n },\n {\n \"type\": \"datasource\",\n \"id\": \"prometheus\",\n \"name\": \"Prometheus\",\n \"version\": \"5.0.0\"\n }\n ],\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"gnetId\": null,\n \"graphTooltip\": 0,\n \"id\": null,\n \"links\": [],\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_YAIS}\",\n \"fill\": 1,\n \"gridPos\": {\n \"h\": 9,\n \"w\": 12,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 8,\n \"legend\": {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"rate(yais_inference_load_ratio_count[5s])\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n },\n {\n \"expr\": \"irate(yais_inference_load_ratio_count[5s])\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"B\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Inference Rate (FPS)\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_YAIS}\",\n \"fill\": 1,\n \"gridPos\": {\n \"h\": 9,\n \"w\": 12,\n \"x\": 12,\n \"y\": 0\n },\n \"id\": 4,\n \"legend\": {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"yais_gpus_power_usage\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"GPU Power (Watts)\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_YAIS}\",\n \"fill\": 1,\n \"gridPos\": {\n \"h\": 9,\n \"w\": 12,\n \"x\": 0,\n \"y\": 9\n },\n \"id\": 2,\n \"legend\": {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"yais_executor_queue_depth\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Queue Depth\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n }\n ],\n \"schemaVersion\": 16,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\": \"now-15m\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"\",\n \"title\": \"YAIS\",\n \"uid\": \"3WLjQkdmk\",\n \"version\": 1\n}\n" }, { "path": "examples/90_Kubernetes/prometheus/yais-metrics.yml", "content": "#\n# Create a Service Account, Role, Role Binding\n# YAIS Specific Prometheus (via Operator) and Service\n#\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: prometheus\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRole\nmetadata:\n name: prometheus\nrules:\n- apiGroups: [\"\"]\n resources:\n - nodes\n - services\n - endpoints\n - pods\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources:\n - configmaps\n verbs: [\"get\"]\n- nonResourceURLs: [\"/metrics\"]\n verbs: [\"get\"]\n---\napiVersion: rbac.authorization.k8s.io/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: prometheus\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: prometheus\nsubjects:\n- kind: ServiceAccount\n name: prometheus\n namespace: default\n---\napiVersion: monitoring.coreos.com/v1\nkind: Prometheus\nmetadata:\n name: yais-metrics\nspec:\n serviceAccountName: prometheus\n serviceMonitorSelector:\n matchLabels:\n scrape: yais\n resources:\n requests:\n memory: 400Mi\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: yais-metrics\nspec:\n ports:\n - name: web\n port: 9090\n selector:\n prometheus: yais-metrics\n" }, { "path": "examples/90_Kubernetes/yais-deploy.yml", "content": "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: yais-example\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: yais-example\n template:\n metadata:\n labels:\n app: yais-example\n annotations:\n sidecar.istio.io/inject: \"true\"\n spec:\n containers:\n - name: yais-example\n image: yais\n command: [\"/work/examples/90_Kubernetes/deploy/build-and-run.sh\"]\n imagePullPolicy: IfNotPresent # Always\n env:\n - name: YAIS_CONCURRENCY\n value: \"8\"\n resources:\n limits:\n nvidia.com/gpu: 1\n ports:\n - name: grpc\n containerPort: 50051\n - name: metrics\n containerPort: 50078\n livenessProbe:\n tcpSocket:\n port: 50051\n initialDelaySeconds: 5\n periodSeconds: 5\n readinessProbe:\n tcpSocket:\n port: 50051\n initialDelaySeconds: 5\n periodSeconds: 5\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: yais-example\n labels:\n app: yais-example\nspec:\n selector:\n app: yais-example\n ports:\n - name: grpc\n port: 50051\n targetPort: grpc\n - name: metrics\n port: 50078\n targetPort: metrics\n---\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n name: yais-example\n labels:\n scrape: yais\nspec:\n selector:\n matchLabels:\n app: yais-example\n endpoints:\n - port: metrics\n interval: 2s\n honorLabels: true\n---\napiVersion: networking.istio.io/v1alpha3\nkind: Gateway\nmetadata:\n name: yais-gateway\nspec:\n selector:\n istio: ingressgateway # use istio default controller\n servers:\n - hosts:\n - \"*\"\n port:\n name: grpc\n number: 80\n protocol: grpc\n---\napiVersion: networking.istio.io/v1alpha3\nkind: VirtualService\nmetadata:\n name: yais-virtual-service\nspec:\n hosts:\n - \"*\"\n gateways:\n - yais-gateway\n http:\n - match:\n - uri:\n prefix: /\n route:\n - destination:\n host: yais-example.default.svc.cluster.local\n port:\n number: 50051\n" }, { "path": "examples/91_Prometheus/README.md", "content": "# Prometheus\n\nWIP\n" }, { "path": "examples/91_Prometheus/scrape.conf", "content": "[[inputs.prometheus]]\n urls = [\"http://localhost:50078/metrics\"]\n[[outputs.file]]\n files = [\"stdout\"]\n" }, { "path": "examples/97_SingleProcessMultiSteam/launch_service.sh", "content": "#!/bin/bash -e\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\ncleanup() {\n kill $(jobs -p) ||:\n}\ntrap \"cleanup\" EXIT SIGINT SIGTERM\n\nENG=${1:-/work/models/ResNet-50-b1-fp32.engine}\nNCTX=${2:-1}\n\nif [ ! -e $ENG ]; then\n echo \"$ENG not found\"\n exit 911\nfi\n\nport=50051\n/work/build/examples/02_TensorRT_GRPC/inference-grpc.x --port=$port --engine=${ENG} --contexts=$NCTX &\nwait-for-it.sh localhost:$port --timeout=0 -- echo \"YAIS Service is ready.\" > /dev/null 2>&1\n\necho \"warmup with client-async.x\"\n/work/build/examples/02_TensorRT_GRPC/client-async.x --count=1000 --port=$port\n\necho\necho \"Starting a shell keeping the services and load-balancer running...\"\necho \"Try /work/build/examples/02_TensorRT_GRPC/siege.x --rate=2000 --port=$port\"\nbash --rcfile <(echo \"PS1='$NCTX x $ENG Subshell: '\")\n" }, { "path": "examples/98_MultiProcessSingleStream/README.md", "content": "# MPS Examples\n\n`run_throughput_test ncopies batch_size engine_file MPS/NOMPS`\n\nV100 - 16GB - DGX-1V\n\nProcesses | MPS | FPS | Batch | Model\n--------- | --- | --- | ----- | -----\n1 | N | 383 | 1 | RN50\n8 | N | 365 | 1 | RN50\n8 | Y | 929 | 1 | RN50\n\n```\nroot@dgx11:/work/src/Examples/98_MultiProcessSingleStream# ./run_throughput_test 8 1 /work/models/ResNet-50-b1-fp32.engine MPS\nstarting 8 inference services\nstarting load balancer\nload balancing over ports: ['50051', '50052', '50053', '50054', '50055', '50056', '50057', '50058']\nrunning test client\n1000 requests in 1.07632seconds; inf/sec: 929.095\n\nroot@dgx11:/work/src/Examples/98_MultiProcessSingleStream# ./run_throughput_test 8 1 /work/models/ResNet-50-b1-fp32.engine NOMPS\nstarting 8 inference services\nstarting load balancer\nload balancing over ports: ['50051', '50052', '50053', '50054', '50055', '50056', '50057', '50058']\nrunning test client\n1000 requests in 2.74228seconds; inf/sec: 364.66\n\nroot@dgx11:/work/src/Examples/98_MultiProcessSingleStream# ./run_throughput_test 1 1 /work/models/ResNet-50-b1-fp32.engine NOMPS\nstarting 1 inference services\nstarting load balancer\nload balancing over ports: ['50051']\nrunning test client\n1000 requests in 2.60915seconds; inf/sec: 383.267\n```\n" }, { "path": "examples/98_MultiProcessSingleStream/run_latency_test", "content": "#!/bin/bash -e\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\ntrap 'kill $(jobs -p)' EXIT\n\nBS=${1:-1}\nENG=${2:-/work/models/ResNet-50-b2-int8.engine}\n\nif [ ! -e $END ]; then\n echo \"$ENG not found\"\n exit 911\nfi\n\necho \"starting inference service\"\nport=50100\n/work/build/examples/02_TensorRT_GRPC/inference-grpc.x --port $port --engine=${ENG} & # > /dev/null 2>&1 &\nwait-for-it.sh localhost:$port --timeout=0 -- echo \"Server ${i} is ready.\" > /dev/null 2>&1\n\necho \"running latency client\"\n/work/build/examples/02_TensorRT_GRPC/client-sync.x --port=$port\n\n" }, { "path": "examples/98_MultiProcessSingleStream/run_throughput_test", "content": "#!/bin/bash -e\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\ncleanup() {\n kill $(jobs -p) ||:\n echo quit | nvidia-cuda-mps-control > /dev/null 2>&1 ||:\n}\ntrap \"cleanup\" EXIT SIGINT SIGTERM\n\nNCTX=${1:-1}\nBS=${2:-1}\nENG=${3:-/work/models/ResNet-50-b1-fp32.engine}\nMPS=${4:-\"MPS\"}\n\nif [ ! -e $ENG ]; then\n echo \"$ENG not found\"\n exit 911\nfi\n\nif [ \"$MPS\" = \"MPS\" ]; then\n nvidia-cuda-mps-control -d ||:\nfi\n\nsleep 1\n\necho \"starting $NCTX inference services\"\nfor i in $(seq 1 $NCTX);\ndo\n port=$(echo \"50050 + $i\" | bc)\n /work/build/examples/02_TensorRT_GRPC/inference-grpc.x --port $port --engine=${ENG} > /dev/null 2>&1 &\n wait-for-it.sh localhost:$port --timeout=0 -- echo \"Server ${i} is ready.\" > /dev/null 2>&1\ndone\n\necho \"starting load balancer\"\n../99_LoadBalancer/run_loadbalancer.py -n $NCTX \n# envoy -c /tmp/lb-envoy.yaml --disable-hot-restart > /dev/null 2>&1 &\nenvoy -c /tmp/lb-envoy.yaml > /dev/null 2>&1 &\nwait-for-it.sh localhost:50050 --timeout=0 -- echo \"Load balancer is ready.\" > /dev/null 2>&1\n\necho \"running test client\"\n/work/build/examples/02_TensorRT_GRPC/client-async.x --count=1000 --port=50050\n\necho\necho \"Starting a shell keeping the services and load-balancer running...\"\necho \"Try /work/build/examples/02_TensorRT_GRPC/siege.x --rate=2000 --port=50050\"\nbash --rcfile <(echo \"PS1='Throughput Subshell: '\")\n" }, { "path": "examples/98_MultiProcessSingleStream/setup.py", "content": "#!/usr/bin/env python3\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\nimport os\nimport subprocess\n\nmodels = [\n (\"ResNet-50-deploy.prototxt\", \"prob\"),\n# (\"ResNet-152-deploy.prototxt\", \"prob\"),\n]\n\nprecisions = [\n (\"fp32\", \"\"),\n (\"fp16\", \"--fp16\"),\n (\"int8\", \"--int8\")\n]\n\ndef main():\n for model, o in models:\n for name, p in precisions:\n for b in [1]: #, 2, 4, 8]:\n n = \"b{}-{}\".format(b, name)\n e = model.replace(\"prototxt\", \"engine\")\n e = e.replace(\"deploy\", n)\n m = os.path.join(\"/work/models\", model)\n if os.path.isfile(e):\n continue\n subprocess.call(\"giexec --deploy={} --batch={} --output={} {} --engine={}\".format(\n m, b, o, p, e\n ), shell=True)\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "examples/99_LoadBalancer/README.md", "content": "# Envoy Load Balancer\n\nVery basic Envoy Proxy L7 load balancer for testing purposes.\n\n`run_loadbalancer.py -n ` will start a copy of envoy \nlistening on port `50050` and load-balancing over ports `[50051, 50051+n-1]`.\n\nYou are responsible for spinning up the backend services.\n\n## Notes\n\nThe load-balancer overhead appears to be about 150us. Running the `client-sync.x`\ndirectly to a backend vs. through the load-balancer shows about 150us overhead per\ntransaction.\n\n```\n# direct\nThroughput Subshell: /work/build/examples/02_TensorRT_GRPC/client-sync.x --port 50051\n1000 requests in 2.69029 seconds; inf/sec: 371.707\n\n# proxied via envoy load-balancer\nThroughput Subshell: /work/build/examples/02_TensorRT_GRPC/client-sync.x --port 50050\n1000 requests in 2.8411 seconds; inf/sec: 351.977\n```" }, { "path": "examples/99_LoadBalancer/lb-envoy.j2", "content": "{#-\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n-#}\nstatic_resources:\n listeners:\n - address:\n socket_address:\n address: 0.0.0.0\n port_value: 50050\n filter_chains:\n - filters:\n - name: envoy.http_connection_manager\n config:\n codec_type: auto\n stat_prefix: ingress_http\n route_config:\n name: local_route\n virtual_hosts:\n - name: backend\n domains:\n - \"*\"\n routes:\n - match:\n prefix: \"/\"\n headers:\n - name: content-type\n value: application/grpc\n route:\n cluster: inference\n http_filters:\n - name: envoy.router\n config: {}\n clusters:\n - name: inference\n connect_timeout: 0.25s\n type: strict_dns\n lb_policy: round_robin\n http2_protocol_options: {}\n hosts:\n{% for port in ports %}\n - socket_address:\n address: 127.0.0.1\n port_value: {{ port }}\n{% endfor %}\n\nadmin:\n access_log_path: \"/dev/null\"\n address:\n socket_address:\n address: 0.0.0.0\n port_value: 8001\n" }, { "path": "examples/99_LoadBalancer/run_loadbalancer.py", "content": "#!/usr/bin/env python3\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\nimport os\nimport inspect\nimport shutil\nimport tempfile\n\nimport click\nfrom jinja2 import Environment, FileSystemLoader, Template\n\ndef render(template_path, data=None, extensions=None, strict=False):\n data = data or {}\n extensions = extensions or []\n env = Environment(\n loader=FileSystemLoader(os.path.dirname(template_path)),\n extensions=extensions,\n keep_trailing_newline=True,\n )\n if strict:\n from jinja2 import StrictUndefined\n env.undefined = StrictUndefined\n\n # Add environ global\n env.globals['environ'] = os.environ.get\n\n return env.get_template(os.path.basename(template_path)).render(data)\n\nscript_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\nFileType = click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True)\n\n@click.command()\n@click.option(\"-n\", default=1)\n@click.option(\"--template\", type=FileType, default=os.path.join(script_path, \"lb-envoy.j2\"))\ndef main(n, template):\n envoy = shutil.which(\"envoy\")\n if not os.path.isfile(envoy):\n raise RuntimeError(\"envoy executable not found in currently directory: {}\".format(envoy))\n ports = [50051 + p for p in range(n)]\n print(\"load balancing over ports: \", [str(p) for p in ports])\n with open(\"/tmp/lb-envoy.yaml\", \"w\") as file:\n file.write(render(template, data={\"ports\": ports}))\n# os.system(\"{} -c /tmp/lb-envoy.yaml\".format(envoy))\n\nif __name__ == \"__main__\":\n main()\n \n" }, { "path": "examples/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nfind_package(gflags 2.2.1 REQUIRED)\n\nadd_subdirectory(nvRPC)\nadd_subdirectory(Deployment)\n\nadd_subdirectory(00_TensorRT)\nadd_subdirectory(01_Basic_GRPC)\nadd_subdirectory(02_TensorRT_GRPC)\n#add_subdirectory(03_Batching)\nadd_subdirectory(04_Middleman)\nadd_subdirectory(10_Internals)\nadd_subdirectory(11_Protos)\nadd_subdirectory(12_FlatBuffers)\n#add_subdirectory(30_PyTensorRT)\n" }, { "path": "examples/Deployment/CMakeLists.txt", "content": "add_subdirectory(ImageClient)\nadd_subdirectory(RouteRequests)\n" }, { "path": "examples/Deployment/ImageClient/CMakeLists.txt", "content": "set(protobuf_MODULE_COMPATIBLE TRUE)\nfind_package(Protobuf CONFIG REQUIRED)\nmessage(STATUS \"Using protobuf ${protobuf_VERSION}\")\nset(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)\nset(_PROTOBUF_PROTOC $)\n\nINCLUDE(GRPCGenerateCPP)\n\nPROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS \n api.proto\n)\n\nPROTOBUF_GENERATE_GRPC_CPP(PROTO_GRPC_SRCS PROTO_GRPC_HDRS \n api.proto\n)\n\nadd_library(deploy-image-client-protos\n ${PROTO_SRCS}\n ${PROTO_GRPC_SRCS}\n)\n\ntarget_link_libraries(deploy-image-client-protos\n PUBLIC\n ${_PROTOBUF_LIBPROTOBUF}\n)\n\ntarget_include_directories(deploy-image-client-protos PUBLIC\n ${CMAKE_CURRENT_BINARY_DIR}\n)\n\npybind11_add_module(deploy_image_client\n client.cc\n)\n\ntarget_link_libraries(deploy_image_client\nPUBLIC\n nvrpc-client\n deploy-image-client-protos\n)\n" }, { "path": "examples/Deployment/ImageClient/api.proto", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n\nsyntax = \"proto3\";\n\npackage trtlab.deploy.image_client;\n \nservice Inference {\n rpc Classify (ImageInfo) returns (Classifications) {}\n rpc Detection (ImageInfo) returns (Detections) {}\n\n rpc ClassifyStream (stream ImageInfo) returns (stream Classifications) {}\n rpc DetectionSream (stream ImageInfo) returns (stream Detections) {}\n}\n \nmessage ImageInfo {\n string api_key = 1;\n string image_uuid = 2;\n string model_name = 3;\n}\n\nmessage Classifications\n{\n string image_uuid = 1;\n repeated Score scores =2;\n Details details = 3;\n}\n\nmessage Detections\n{\n string image_uuid = 1;\n repeated Object objects = 2;\n Details details = 3;\n}\n\nmessage Score\n{\n int32 class_idx = 1;\n float score = 2;\n}\n\nmessage Object\n{\n string class_name = 1;\n float upper_left_x = 2; // [0, 1] normalized on width\n float upper_left_y = 3; // [0, 1] normalized on height\n float width = 4; // [0, 1] normalized on width\n float height = 5; // [0, 1] normalized on height\n}\n\n// this is where you can customize what details your service\n// returns to the user\nmessage Details\n{\n float request_time = 1;\n float compute_time = 2;\n}" }, { "path": "examples/Deployment/ImageClient/client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n\n#include \"client.h\"\n\nusing namespace trtlab;\nusing namespace nvrpc;\n\nnamespace py = pybind11;\n\nusing deploy::image_client::Classifications;\nusing deploy::image_client::Detections;\nusing deploy::image_client::ImageInfo;\nusing deploy::image_client::Inference;\n\nImageClient::ImageClient(std::string hostname)\n{\n auto executor = std::make_shared(1);\n\n auto channel = grpc::CreateChannel(hostname, grpc::InsecureChannelCredentials());\n std::shared_ptr stub = Inference::NewStub(channel);\n\n auto classify_prepare_fn = [stub](::grpc::ClientContext * context, const ImageInfo& request,\n ::grpc::CompletionQueue* cq) -> auto\n {\n return std::move(stub->PrepareAsyncClassify(context, request, cq));\n };\n\n auto detection_prepare_fn = [stub](::grpc::ClientContext * context, const ImageInfo& request,\n ::grpc::CompletionQueue* cq) -> auto\n {\n return std::move(stub->PrepareAsyncDetection(context, request, cq));\n };\n\n m_ClassifyClient = std::make_unique>(\n classify_prepare_fn, executor);\n m_DetectionClient = std::make_unique>(\n detection_prepare_fn, executor);\n}\n\nstd::shared_future ImageClient::Classify(const std::string& model_name,\n const std::string& image_uuid)\n{\n ImageInfo image_info;\n image_info.set_model_name(model_name);\n image_info.set_image_uuid(image_uuid);\n std::map headers = {{\"custom-metadata-model-name\", model_name}};\n auto post = [](ImageInfo& input, Classifications& output,\n ::grpc::Status& status) -> ClassifyResult {\n ClassifyResult results(output);\n return std::move(results);\n };\n return m_ClassifyClient->Enqueue(std::move(image_info), post, headers);\n}\n\nstd::shared_future ImageClient::Detection(const std::string& model_name,\n const std::string& image_uuid)\n{\n ImageInfo image_info;\n image_info.set_model_name(model_name);\n image_info.set_image_uuid(image_uuid);\n std::map headers = {{\"custom-metadata-model-name\", model_name}};\n auto post = [](ImageInfo& input, Detections& output,\n ::grpc::Status& status) -> DetectionResult {\n DetectionResult results(output);\n return std::move(results);\n };\n return m_DetectionClient->Enqueue(std::move(image_info), post, headers);\n}\n\nClassifyResult::ClassifyResult(const ::trtlab::deploy::image_client::Classifications& pb)\n : m_UUID(pb.image_uuid())\n{\n}\n\nDetectionResult::DetectionResult(const ::trtlab::deploy::image_client::Detections& pb)\n : m_UUID(pb.image_uuid())\n{\n}\n\nusing PyClassifyFuture = std::shared_future;\nusing PyDetectionFuture = std::shared_future;\n\nPYBIND11_MAKE_OPAQUE(PyClassifyFuture);\nPYBIND11_MAKE_OPAQUE(PyDetectionFuture);\n\nPYBIND11_MODULE(deploy_image_client, m)\n{\n py::class_>(m, \"ImageClient\")\n .def(py::init(), py::arg(\"hostname\") = \"trt.lab\")\n .def(\"classify\", &ImageClient::Classify)\n .def(\"detection\", &ImageClient::Detection);\n\n py::class_>(m, \"ClassifyFuture\")\n .def(\"wait\", &PyClassifyFuture::wait, py::call_guard())\n .def(\"get\", &PyClassifyFuture::get, py::call_guard());\n\n py::class_>(m, \"DetectionFuture\")\n .def(\"wait\", &PyDetectionFuture::wait, py::call_guard())\n .def(\"get\", &PyDetectionFuture::get, py::call_guard());\n\n py::class_>(m, \"ClassifyResult\")\n .def_property_readonly(\"uuid\", &ClassifyResult::UUID);\n\n py::class_>(m, \"DetectionResult\")\n .def_property_readonly(\"uuid\", &DetectionResult::UUID);\n}" }, { "path": "examples/Deployment/ImageClient/client.h", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#pragma once\n#include \n#include \n#include \n\n#include \"nvrpc/client/client_unary.h\"\n\n#include \"api.grpc.pb.h\"\n#include \"api.pb.h\"\n\nclass ClassifyResult\n{\n public:\n ClassifyResult(const ::trtlab::deploy::image_client::Classifications&);\n const std::string& UUID() const { return m_UUID; }\n\n private:\n std::string m_UUID;\n};\n\nstruct DetectionResult\n{\n public:\n DetectionResult(const ::trtlab::deploy::image_client::Detections&);\n const std::string& UUID() const { return m_UUID; }\n\n private:\n std::string m_UUID;\n};\n\nclass ImageClient\n{\n public:\n ImageClient(std::string);\n ~ImageClient() {}\n\n std::shared_future Classify(const std::string&, const std::string&);\n std::shared_future Detection(const std::string&, const std::string&);\n\n private:\n using ImageInfo = ::trtlab::deploy::image_client::ImageInfo;\n using Classifications = ::trtlab::deploy::image_client::Classifications;\n using Detections = ::trtlab::deploy::image_client::Detections;\n\n std::unique_ptr<::nvrpc::client::ClientUnary> m_ClassifyClient;\n std::unique_ptr<::nvrpc::client::ClientUnary> m_DetectionClient;\n};\n" }, { "path": "examples/Deployment/ImageClient/client.py", "content": "## Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n##\n## Redistribution and use in source and binary forms, with or without\n## modification, are permitted provided that the following conditions\n## are met:\n## * Redistributions of source code must retain the above copyright\n## notice, this list of conditions and the following disclaimer.\n## * Redistributions in binary form must reproduce the above copyright\n## notice, this list of conditions and the following disclaimer in the\n## documentation and/or other materials provided with the distribution.\n## * Neither the name of NVIDIA CORPORATION nor the names of its\n## contributors may be used to endorse or promote products derived\n## from this software without specific prior written permission.\n##\n## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n## EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n## PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n## CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n## EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n## PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n## PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n## OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n## (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n## OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\nimport os\nimport uuid\n\nimport boto3\nimport deploy_image_client as client\n\nclass ImageClient:\n\n def __init__(self, *, hostname = \"trt.lab\"):\n self._cpp_client = client.ImageClient(hostname)\n self._s3_client = self._get_s3_client()\n\n def classify(self, image_path, model):\n key = self._upload_to_s3(image_path)\n return self._cpp_client.classify(key, model)\n\n def object_detection(self, image_path, model):\n key = self._upload_to_s3(image_path)\n return self._cpp_client.object_detection(key, model)\n\n def _get_s3_client(self):\n kwargs = {}\n if os.environ.get(\"AWS_ENDPOINT_URL\"):\n kwargs = {\n endpoint_url: os.environ.get(\"AWS_ENDPOINT_URL\"),\n use_ssl: False,\n verify: False,\n }\n return boto3.client(\"s3\", **kwargs)\n\n def _check_if_file(self, file_path):\n if not os.path.isfile(file_path):\n raise RuntimeError(\"{} is not a file\".format(file_path))\n\n def _upload_to_s3(self, image_path):\n self._check_if_file(image_path)\n key = str(uuid.uuid4())\n with open(image_path, \"rb\") as data:\n self._s3_client.upload_fileobj(data, 'images', key)\n return key\n" }, { "path": "examples/Deployment/Kubernetes/basic-trtis-deployment/deploy.yml", "content": "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: basic-trtis-deployment\n namespace: trtlab\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: basic-trtis-deployment\n template:\n metadata:\n labels:\n app: basic-trtis-deployment\n annotations:\n sidecar.istio.io/inject: \"true\"\n spec:\n containers:\n - name: trtis\n image: nvcr.io/nvidia/tensorrtserver:19.02-py3\n command: [\"trtserver\", \"--model-store=/tmp/models\"]\n imagePullPolicy: IfNotPresent\n resources:\n limits:\n nvidia.com/gpu: 1\n ports:\n - name: http\n containerPort: 8000\n - name: grpc\n containerPort: 8001\n - name: metrics\n containerPort: 8002\n livenessProbe:\n httpGet:\n path: /api/health/live\n port: http\n initialDelaySeconds: 10\n periodSeconds: 5\n readinessProbe:\n httpGet:\n path: /api/health/ready\n port: http\n initialDelaySeconds: 10\n periodSeconds: 5\n volumeMounts:\n - mountPath: /tmp/models\n name: model-store\n volumes:\n - name: model-store\n hostPath:\n path: /shared/trtis/example-model-store\n type: Directory\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: basic-trtis-deployment\n namespace: trtlab\n labels:\n app: basic-trtis-deployment\nspec:\n selector:\n app: basic-trtis-deployment\n ports:\n - name: http\n port: 8000\n targetPort: http\n - name: grpc\n port: 8001\n targetPort: grpc\n - name: metrics\n port: 8002\n targetPort: metrics\n" }, { "path": "examples/Deployment/Kubernetes/basic-trtis-deployment/istio-ingress.yml", "content": "---\napiVersion: networking.istio.io/v1alpha3\nkind: Gateway\nmetadata:\n name: basic-trtis-deployment-gateway\n namespace: trtlab\nspec:\n selector:\n istio: ingressgateway\n servers:\n - port:\n name: http\n number: 80\n protocol: grpc\n hosts:\n - \"trt.lab\"\n---\napiVersion: networking.istio.io/v1alpha3\nkind: VirtualService\nmetadata:\n name: basic-trtis-deployment-virtual-service\n namespace: trtlab\nspec:\n hosts:\n - \"trt.lab\"\n gateways:\n - basic-trtis-deployment-gateway\n http:\n - match:\n - uri:\n prefix: /api/health/\n route:\n - destination:\n host: basic-trtis-deployment\n port:\n number: 8000\n - match:\n - uri:\n prefix: /\n route:\n - destination:\n host: basic-trtis-deployment\n port:\n number: 8001\n---\napiVersion: networking.istio.io/v1alpha3\nkind: DestinationRule\nmetadata:\n name: basic-trtis-deployment-load-balancer\n namespace: trtlab\nspec:\n host: basic-trtis-deployment\n trafficPolicy:\n loadBalancer:\n simple: LEAST_CONN\n" }, { "path": "examples/Deployment/Kubernetes/basic-trtis-deployment/scrape-metrics.yml", "content": "---\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n name: basic-trtis-deployment\n namespace: trtlab\n labels:\n scrape: nv-inference-metrics\nspec:\n selector:\n matchLabels:\n app: basic-trtis-deployment\n endpoints:\n - port: metrics\n interval: 2s\n honorLabels: true\n" }, { "path": "examples/Deployment/ObjectStore/README.md", "content": "# Object Store\n\nIn the Image Service example, the ImageClient separates out an inference request\ninto two components: \n- a bulk data transfers to a backend store, \n- a gRPC request that contains the details of the request (model, file_handle, etc.)\n\nTo implement this concept, we will use an S3-compatible Object Store. The\nexample should work equally well on AWS S3 or via Rook's S3 CephObjectStore\nimplementation running locally a Kubernetes cluster. For more details on\nhow Kubernetes and Rook were installed, see the [NVIDIA DeepOps Project](https://github.com/nvidia/deepops).\n\nThis folder contains some basic configuration files and scripts for preparing the\nObjectStore for our Image Service.\n\n## AWS S3\n\nSimply sent your AWS API configuration or export the following environment variables.\n\n```\nexport AWS_ACCESS_KEY_ID=\nexport AWS_SECRET_ACCESS_KEY=\n```\n\n## Rook + Kubernetes\n\nYou will need to modify some of the configuration files for your cluster.\n\n- `rook-s3.yml` options:\n - requires 3 unique hosts with bluestore backed OSDs\n - creates a `trtlab` user\n\nIf you modify the name of the ObjectStore (`trtlab-s3`) and or the username (`trtlab`),\nbe aware the `get_rook_s3_keys.sh` needs to be modified.\n\nSimilarly, the examples uses `s3.trt.lab` as the endpoint on which the storage\nis hosted. If you change this, you will need to modify `get_rook_s3_keys.sh` to\noutput the proper `AWS_ENDPOINT_URL`. You will also need to modify the ingress\nexamples with the proper hostname.\n\n```\nkubectl apply -f rook-s3.yml\nkubectl apply -f ingress-nginx.yml\n```\n\n### Setup your environment\n```\neval $(./get_rook_s3_keys.sh)\n```\n\n### Prepare your Image bucket\n\nNote: you will need to have python3 and boto installed. This does not have be done\ninside the container.\n\n```\npython3 create_buckets.py\n```\n\n## TODOs\n\n- [ ] Export S3 keys to a Kubernetes Secret\n- [ ] Scripts for bucket maintenance: probably some k8s CronJobs\n- [ ] Update Istio ingress example" }, { "path": "examples/Deployment/ObjectStore/create_buckets.py", "content": "import os\nimport boto3\n\ns3 = boto3.client(\"s3\", use_ssl=False, verify=False, \n endpoint_url=os.environ.get(\"AWS_ENDPOINT_URL\"))\n\nresponse = s3.list_buckets()\n\nbuckets = [b[\"Name\"] for b in response[\"Buckets\"]]\n\nif \"images\" not in buckets:\n s3.create_bucket(Bucket=\"images\")\n\nresponse = s3.list_buckets()\nbuckets = [b[\"Name\"] for b in response[\"Buckets\"]]\n\nprint(buckets)\n" }, { "path": "examples/Deployment/ObjectStore/get_rook_s3_keys.sh", "content": "#!/bin/bash\nobjstore=trtlab-s3\nuser=trtlab\necho -n export AWS_ACCESS_KEY_ID=\nkubectl -n rook-ceph get secret rook-ceph-object-user-${objstore}-${user} -o yaml | grep AccessKey | awk '{print $2}' | base64 --decode\necho\necho -n export AWS_SECRET_ACCESS_KEY=\nkubectl -n rook-ceph get secret rook-ceph-object-user-${objstore}-${user} -o yaml | grep SecretKey | awk '{print $2}' | base64 --decode\necho\necho export AWS_ENDPOINT_URL=http://s3.trt.lab\necho\n" }, { "path": "examples/Deployment/ObjectStore/ingress-istio.yml", "content": "# not working yet\n---\napiVersion: networking.istio.io/v1alpha3\nkind: Gateway\nmetadata:\n name: trtlab-s3-gateway\nspec:\n selector:\n app: trtlab-s3\n istio: ingressgateway\n servers:\n - hosts:\n - \"s3.trt.lab\"\n port:\n number: 80\n name: http\n protocol: HTTP\n---\napiVersion: networking.istio.io/v1alpha3\nkind: VirtualService\nmetadata:\n name: trtlab-s3-virtualservice\nspec:\n hosts:\n - \"s3.trt.lab\"\n gateways:\n - trtlab-s3-gateway\n http:\n - match:\n - uri:\n prefix: /\n route:\n - destination:\n host: rook-ceph-rgw-trtlab-s3.rook-ceph\n port:\n number: 80\n" }, { "path": "examples/Deployment/ObjectStore/ingress-nginx.yml", "content": "apiVersion: extensions/v1beta1\nkind: Ingress\nmetadata:\n annotations:\n nginx.ingress.kubernetes.io/proxy-body-size: \"0\"\n nginx.ingress.kubernetes.io/proxy-read-timeout: \"600\"\n nginx.ingress.kubernetes.io/proxy-send-timeout: \"600\"\n name: trtlab-s3-ingress\n namespace: rook-ceph\nspec:\n rules:\n - host: s3.trt.lab\n http:\n paths:\n - backend:\n serviceName: rook-ceph-rgw-trtlab-s3\n servicePort: 80\n path: /\n" }, { "path": "examples/Deployment/ObjectStore/rook-s3.yml", "content": "---\napiVersion: ceph.rook.io/v1\nkind: CephObjectStore\nmetadata:\n name: trtlab-s3\n namespace: rook-ceph\nspec:\n metadataPool:\n failureDomain: host\n replicated:\n size: 3\n dataPool:\n failureDomain: host\n erasureCoded:\n dataChunks: 2\n codingChunks: 1\n gateway:\n type: s3\n sslCertificateRef:\n port: 80\n securePort:\n instances: 1\n allNodes: false\n---\napiVersion: ceph.rook.io/v1\nkind: CephObjectStoreUser\nmetadata:\n name: trtlab\n namespace: rook-ceph\nspec:\n store: trtlab-s3\n displayName: \"TensorRT Laboratory\"\n" }, { "path": "examples/Deployment/README.md", "content": "# Deploying Inference Services\n\nThis document/example folder is a work in progress. Its intent is to cover\nvarious aspect of deployment including strategies, limitations, services and\nkubernetes examples of deploying inference services.\n\nOver the course of this guide, we will build a full end-to-end image processing\nservice deployed on Kubernetes.\n\nLet us start by assuming all your models can be served/deployed with the\n[TensorRT Inference Server, aka TRTIS](https://github.com/nvidia/tensorrt-inference-server).\nOne of the primary advantages of TRTIS is the ability to host multiple models in\na single linux process. Given the capabilities of modern GPUs, this is the most\nefficient way to for multiple models to efficiently share of both compute and\nmemory resources.\n\nNext, lets dive a little deeper into the features of TRTIS that gives this effiency advantage.\n\n - _Concurrent Executions_ allow for multiple independent inference batches to be\n in-flight on the device at a given time. This could be multiple batches of\n the same model or single batches of different models, or any combination\n imaginable. \n - Running on a Tesla V100 GPU with ResNet-152 Batch8\n - Allowing only 1 in-flight batch8 yields XXX images/sec with a compute\n latency of YYY.\n - Allowing 8 concurrency batch8 executions increases the throughput to\n 2500 images/sec; however, the compute latency per batch increases to\n ZZZ.\n - To evaluate the performance of TensorRT models a function of concurrent\n in-flight executions we provide the [TensorRT/ConcurrentExecution](../TensorRT/ConcurrentExecution)\n example.\n ```bash\n infer.x --engine=/external/models/ResNet-152-b8-fp16.engine --concurrency=8\n ```\n - The value of concurrent executions will differ depending on the compute\n requirements of the model and the GPU on which it executes. The best\n practice is to benchmark and evaluates performance.\n\n - _Tunable Concurency_ enables you to specify on a per model basis the number\n of concurreny copies that can be executed at any given time. Follow the guidelines\n Concurrent Executions to tune this option on a per model basis.\n\n - _Dynamic Batching_ allows individual requests from either same or different\n clients to be multiplexed into a single mini-batch and infered. Dynamic\n Batching is performed on a per-model basis and can have a max\n `preferred_batch_size` and a `max_queue_delay_microseconds` which specific\n how long of a delay messages are allowed to accumulate. Batching is one of\n the best ways to improve throughput. Depending on your needs, you will want\n to balance the added latency required for batching vs the throughput\n improvements achieved.\n - Note: in scale-out deployments where unary (send/recv, not streaming)\n requests are being load-balanced across multiple TRTIS instances, the\n value of dynamic batching in the TRTIS decreases as the number of replicas \n increases.\n - TODO: We address this issue by creating Dynamic Batching Services that sit\n infront of the Load-Balancer. Add a discussion and update the [examples/03_Batching]\n example with the latest streaming Server/Client\n\n - _Custom Metrics_ provide application-level metrics on how the TRTIS service\n is performing. Analyzing these metrics can provide insight on when a service\n is being overloaded and when to add more resources. TRTIS and some TRTLAB\n examples expose Prometheus Metrics.\n - [examples/90_Kubernetes/prometheus] is one exmaple of how to use\n Kubernetes + Prometheus + Grafana to scrape and visualize metrics from\n running TRTIS services.\n - TODO: Document and clean up examples.\n\nThe goal of this project is to provide supplementary support to TRTIS by\nproviding buildings blocks that help you build companion microservices that\nworth with TRTIS, as well as example deployment scenarios.\n\n## Configure a TRTIS Kubernetes Deployment\n\nIn Kubernetes, a\n[Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/)\nallows you to define a [Pod](https://kubernetes.io/docs/concepts/workloads/pods/pod/)\nand the number of copies of that pod, i.e.\n[ReplicaSet](https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/)\nthat you would like deployed on your cluster.\n\n### Build a Model Store\n\nThe [TRTIS Model Store] is the collection of models that will be loaded and\nserved by a TRTIS instance. For details on how to build a model store, see the [Deployment/ModelStore]\nexample.\n\nIn order to define our Kubernetes Deployment, we need to map our model store\ninto our TRTIS Pod. There are multiple ways this can be achieved.\n- Extend the TRTIS container image and add the model store to the image\n- Mount an External Volume into the TRTIS Pod\n- Use another container in the same TRTIS Pod that will dynamically generate a\n model store from some external data sources, e.g. S3.\n - TODO: Add this example\n\nMy Kubernetes cluster has an NFS mount at `/shared` on every node. For this\nexample, our model store will be located at `/shared/trtis/example-model-store`\n\n### Deploy TRTIS Pods\n\n\n\n\n## Configure Multiple TRTIS Deployments with Different Sets of Models\n\nNow suppose you are serving more models than you can allocate on a single GPU. In this scenario, we can split our list of models into groups and spread those groups out over multiple TRTIS `Deployments`\n\n## Scenario #1: 2 Models, 10 Servers, 20 GPUs\n\nAssume you have two computer vision models, e.g. classification, object\ndetection, segmentation, that you wish to deploy on 10 servers.\n\nProbably the first questions you might ask yourself is what is the breakdown of\nload expected for each model. Is it 50/50? Or is it 90/10? Does it vary by time\nof day? Is it predictive?\n\nThe strategy with TRTIS is simple, you deploy 20 replicas of the TRTIS service\nacross 10 servers, 1 TRTIS service per GPU and tell your load-balancer to\nround-robin requests across your services.\n\nHowever, unless you customize TRTIS, by default, TRTIS only accepts raw tensors\nas inputs and returns raw tensors an outputs.\n\nWhat are your inputs? And what's a reasonable expectation for the rest of the\ninference pipeline?\n\nLet's assume you are receiving JPEG images. First, you need to decode the images to\nraw pixels, then you need to prepare the images to be inferred. To keep things\nsimple, let's assume that both models use the same input preprocessing method.\n\nWhat is the compression ratio of your JPEG images? Assuming the images are\n8-bit, then\n[a blog post by Graphics Mill](https://www.graphicsmill.com/blog/2014/11/06/Compression-ratio-for-different-JPEG-quality-values#.XHtdPpNKiXE_)\nmeasured the compression ratio to be 1:5.27 for JPEG Quality 100 (Q=100) and\n1:43.27 for Q=55. For this discussion we wil focus on Q=80, which was measured\nto be a 1:25 ratio. This is the decompression ratio for JPEG bytes to INT8 pixel\nvalues. However, most DNN models after normalization accept tensors that are\nfp32 values as inputs. This means that we have a 1:100 an increased ratio of\ninput bytes to bytes of input tensors.\n\nA 100KiB image becomes a 9.75MiB data structure that needs to be provided to\nTRTIS. This 100x increase in size is big, but not unreasonably large. If your\nimages are coming in over the Internet/WAN, then your LAN connction is likely to\nbe 100x faster. However, if this was video, that compression ratio per frame\nwould be MUCH larger. To future-proof our implementation, we are going to do our\nbest to minimize the amount of data we move around.\n\nNext, what is it that our users are providing to us beside just the image to be\ninferred. The client must inform the server which model should be used for the\nrequest. And similarly, the server probably also wants to know some details\nabout the user. In this example, we will using a user API key to authenticate\nthe user.\n\nNow let's look at the data payload the client will be sending our server. We are\ngoing to break down an image inference request into two parts:\n- the bulk image data\n- the request metadata: api key, image_uuid, model_name\n\nThis separation allows us to commmunicate and move each component more\noptimally. We will send the JPEG image bytes directly to an object store and we\nwill use a gRPC message to communicate the metadata for the request.\n\nWhy not embed the image directly into the gRPC message? While this is certainly\na possibility, we are chosing this separation because in our optimial pipeline,\nour client request message will go through several services before the bulk\nimage data is needed. These services include an ingress router/load-balancer and\nan external batching service before they are sent to the image pre-processing\nservice. This separate helps us future-proof our implementation by avoiding\nunnecessary data movement.\n\nLet's break down our client implementation:\n- 1) writes the jpeg image data to an S3-compatible object store as some UUID.jpg\n- 2) creates an async gRPC unary (send/recv) RPC request to our inference service.\n - the message payload consists of our client api key, image uuid and model_name\n - add custom headers that will enable our ingress router/load-balancer to\n properly route the message to the correct target without the need to\n deserialize the request payload. this allows us to route message directly to\n services specify to some metadata. In this case, we will add the model name\n to the headers so we can eventually route requests to batching services\n unique to that model.\n\nThis means our client can continue to issue async inference requests with the\npromise that the results will returned in some future time.\n\nOn the server side, this data move break down the inference request into two\ncomponents, the data and the request.\n\nLet's assume our incoming data transport is very effficient. The images will\nbe deposited into an S3-compatible object store. On my Kubernetes cluster,\nI'll be using [Rook's Mino/S3 Operator](https://rook.io), but it would work\nequally well on AWS.\n\n" }, { "path": "examples/Deployment/RouteRequests/CMakeLists.txt", "content": "add_executable(test_image_service.x\n test_service.cc\n)\n\ntarget_link_libraries(test_image_service.x\nPUBLIC\n nvrpc\n deploy-image-client-protos\n gflags\n)\n" }, { "path": "examples/Deployment/RouteRequests/README.md", "content": "# Routing Requests\n\nIf we have multiple instances of TRTIS each with different models, we need a way\nto route requests to the proper service. There are two convenient options:\nrouting by subdomain or routing by headers.\n\nIn this example, we will have three unique pools of TRTIS services:\n- Pool A: only handles `model_a` `Classify` requests\n- Pool B: only handles `model_b` requests (`Classify` or `Detection`)\n- General Pool: handles all other requests\n\nIn our hypothetical deployment scenario, both `model_a` and `model_b` are\nparticularly active so we have dedicated resources to handle those requests.\nSimilarly, our general pool has it's own fixed size. Later, we will show out to\nauto-scale pods based on TRTIS and GPU metrics.\n\nA simple approach would be to host `model_a.trt.lab`, `model_b.trt.lab` and\n`general_pool.trt.lab` and have the client make the decision sender side on\nwhere to send the requests. However, changest to the service layout would\nrequire updates to the clients software which makes this option less appealing.\n\nIdeally, we want our entire service to be hosted on a single endpoint `trt.lab`. \n\nTo ensure our requests arrive at the proper destination server-side, we have our\nclient add [Custom Metadata](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md) to each gRPC request.\n\n```c++\nstd::map headers = {{\"custom-metadata-model-name\", model_name}};\n```\n\nwhich inside our client library\n\n```c++\nfor (auto& header : headers)\n{\n // add headers to ::grpc::ClientContext\n ctx->m_Context.AddMetadata(header.first, header.second);\n}\n```\n\nTo test routing, we have provided an `envoy_config.yaml` configuration. The\nload-balancer/router listens on port 50050 and routes to three sample services\nrunning on 51051, 51052 and 51053.\n\nTo differentiate by endpoint, `Classify` or `Detection`, we can match routes on their `uri`.\n\nHere is the relevant parts of the envoy config:\n```yaml\n- match:\n prefix: /trtlab.deploy.image_client.Inference/Classify\n headers:\n - name: custom-metadata-model-name\n exact_match: model_a\n grpc:\n route:\n cluster: classify_model_a\n- match:\n prefix: /\n headers:\n - name: custom-metadata-model-name\n exact_match: model_b\n grpc:\n route:\n cluster: model_b\n- match:\n prefix: /\n grpc:\n route:\n cluster: general_pool\n```\n\n`test_routing.sh` provides a convenient means to test the configuration. It will\ncompile a simple implementation of the ImageClient service, bring up an instance\nof Envoy, 3 instances of the test_service, then send both `Classify` and\n`Detection` requests with three different models at the router. The service\nimplementation simple returns which named service handled the request.\n\n```s\nroot@5e8ffb38df87:/work/examples/Deployment/RouteRequests# ./test_routing.sh \n... some start up output ...\nTesting Classify RPC\nI0307 13:41:36.614954 355 test_service.cc:74] model_a served by model_a\nI0307 13:41:36.616070 359 test_service.cc:74] model_b served by model_b\nI0307 13:41:36.617031 362 test_service.cc:74] model_c served by general_pool\nTesting Detection RPC\nI0307 13:41:36.617636 362 test_service.cc:74] model_a served by general_pool\nI0307 13:41:36.618005 359 test_service.cc:74] model_b served by model_b\nI0307 13:41:36.618367 362 test_service.cc:74] model_c served by general_pool\n\n**** Test Passed ****\n```\n\nWhile we are using Envoy (v1.9) directly in this example, we will later show how\nthis can be accomplished in Istio. A major TODO in this project is to build a\nTRTIS operator while will provide Kubernetes CRD that will be able to\ndynamically manage the routes as a function of where the model will be loaded on\nthe cluster. Whereas this example shows static placement, we eventually want to\nget to fully dynamic routes." }, { "path": "examples/Deployment/RouteRequests/envoy_config.yaml", "content": "static_resources:\n listeners:\n - name: listener_0\n address:\n socket_address: { address: 0.0.0.0, port_value: 50050 }\n filter_chains:\n - filters:\n - name: envoy.http_connection_manager\n config:\n codec_type: auto\n stat_prefix: ingress_http\n route_config:\n name: local_route\n virtual_hosts:\n - name: backend\n domains:\n - \"*\"\n routes:\n - match:\n prefix: /trtlab.deploy.image_client.Inference/Classify\n headers:\n# - name: content-type\n# value: application/grpc\n - name: custom-metadata-model-name\n exact_match: model_a\n grpc:\n route:\n cluster: classify_model_a\n - match:\n prefix: /\n headers:\n# - name: content-type\n# value: application/grpc\n - name: custom-metadata-model-name\n exact_match: model_b\n grpc:\n route:\n cluster: model_b\n - match:\n prefix: /\n# headers:\n# - name: content-type\n# value: application/grpc\n grpc:\n route:\n cluster: general_pool\n http_filters:\n - name: envoy.router\n config: {}\n clusters:\n - name: classify_model_a\n connect_timeout: 0.25s\n type: strict_dns\n lb_policy: round_robin\n http2_protocol_options: {}\n hosts:\n - socket_address:\n address: 127.0.0.1\n port_value: 51051\n - name: model_b\n connect_timeout: 0.25s\n type: strict_dns\n lb_policy: round_robin\n http2_protocol_options: {}\n hosts:\n - socket_address:\n address: 127.0.0.1\n port_value: 51052\n - name: general_pool\n connect_timeout: 0.25s\n type: strict_dns\n lb_policy: round_robin\n http2_protocol_options: {}\n hosts:\n - socket_address:\n address: 127.0.0.1\n port_value: 51053\n\nadmin:\n access_log_path: \"/dev/null\"\n address:\n socket_address:\n address: 0.0.0.0\n port_value: 8001\n" }, { "path": "examples/Deployment/RouteRequests/test_client.py", "content": "import os\n\nimport deploy_image_client as cpp_client\n\n\ndef main():\n if not os.environ.get(\"TRTLAB_ROUTING_TEST\"):\n raise RuntimeError(\n \"Plese run this script in the environment setup by test_routing.sh\")\n router = cpp_client.ImageClient(\"localhost:50050\")\n\n print(\"Testing Classify RPC\")\n\n a = router.classify(\"model_a\", \"via_router_uuid1\").get()\n b = router.classify(\"model_b\", \"via_router_uuid2\").get()\n c = router.classify(\"model_c\", \"via_router_uuid3\").get()\n\n assert a.uuid == \"model_a\"\n assert b.uuid == \"model_b\"\n assert c.uuid == \"general_pool\"\n\n print(\"Testing Detection RPC\")\n\n a = router.detection(\"model_a\", \"via_router_uuid1\").get()\n b = router.detection(\"model_b\", \"via_router_uuid2\").get()\n c = router.detection(\"model_c\", \"via_router_uuid3\").get()\n\n assert a.uuid == \"general_pool\"\n assert b.uuid == \"model_b\"\n assert c.uuid == \"general_pool\"\n\n print(\"\\n**** Test Passed ****\\n\")\n\n\nif __name__ == \"__main__\":\n try:\n main()\n except RuntimeError as e:\n print(\"\\n**** Error ****\")\n print(e)\n print()\n" }, { "path": "examples/Deployment/RouteRequests/test_routing.sh", "content": "#!/bin/bash\n\ncleanup() {\n kill $(jobs -p) ||:\n}\ntrap \"cleanup\" EXIT SIGINT SIGTERM\n\n(cd /work/build/examples/Deployment/ImageClient; make)\n(cd /work/build/examples/Deployment/RouteRequests; make)\n\nexport PYTHONPATH=$PYTHONPATH:/work/build/examples/Deployment/ImageClient\n\nexe=/work/build/examples/Deployment/RouteRequests/test_image_service.x\n\n$exe --hostname=\"model_a\" --ip_port=\"0.0.0.0:51051\" & #> /dev/null 2>&1 &\n$exe --hostname=\"model_b\" --ip_port=\"0.0.0.0:51052\" & #> /dev/null 2>&1 &\n$exe --hostname=\"general_pool\" --ip_port=\"0.0.0.0:51053\" & #> /dev/null 2>&1 &\nenvoy -c envoy_config.yaml > /dev/null 2>&1 &\n\nwait-for-it.sh localhost:50050 --timeout=0 -- echo \"Envoy on 50050 ready\"\nwait-for-it.sh localhost:51051 --timeout=0 -- echo \"ModelA on 51051 ready\"\nwait-for-it.sh localhost:51052 --timeout=0 -- echo \"ModelB on 51052 ready\"\nwait-for-it.sh localhost:51053 --timeout=0 -- echo \"General Pool on 51053 ready\"\n\nexport TRTLAB_ROUTING_TEST=True\n\npython3 test_client.py\n\n" }, { "path": "examples/Deployment/RouteRequests/test_service.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n\n#include \"tensorrt/laboratory/core/pool.h\"\n#include \"tensorrt/laboratory/core/resources.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n\nusing trtlab::Resources;\nusing trtlab::ThreadPool;\n\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n\nusing nvrpc::AsyncRPC;\nusing nvrpc::AsyncService;\nusing nvrpc::Context;\nusing nvrpc::Executor;\nusing nvrpc::Server;\n\n#include \"api.grpc.pb.h\"\n#include \"api.pb.h\"\n\nusing trtlab::deploy::image_client::Classifications;\nusing trtlab::deploy::image_client::Detections;\nusing trtlab::deploy::image_client::ImageInfo;\nusing trtlab::deploy::image_client::Inference;\n\n// CLI Options\nDEFINE_string(hostname, \"localhost\", \"Hostname\");\nDEFINE_string(ip_port, \"0.0.0.0:50051\", \"IP/Port on which to listen\");\n\nclass TestResources : public Resources\n{\n public:\n TestResources(const std::string& hostname) : m_Hostname(hostname) {}\n const std::string& Hostname() const { return m_Hostname; }\n\n private:\n std::string m_Hostname;\n};\n\ntemplate\nclass TestContext final : public Context\n{\n void ExecuteRPC(ImageInfo& input, Output& output) final override\n {\n LOG(INFO) << input.model_name() << \" served by \" << this->GetResources()->Hostname();\n output.set_image_uuid(this->GetResources()->Hostname());\n this->FinishResponse();\n }\n};\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"test_deploy_client\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n Server server(FLAGS_ip_port);\n auto service = server.RegisterAsyncService();\n auto rpc_classify = service->RegisterRPC>(\n &Inference::AsyncService::RequestClassify);\n auto rpc_detection =\n service->RegisterRPC>(&Inference::AsyncService::RequestDetection);\n\n auto resources = std::make_shared(FLAGS_hostname);\n auto executor = server.RegisterExecutor(new Executor(1));\n\n executor->RegisterContexts(rpc_classify, resources, 1);\n executor->RegisterContexts(rpc_detection, resources, 1);\n\n server.Run(std::chrono::milliseconds(2000), [] {});\n}\n" }, { "path": "examples/Deployment/batcher.cc", "content": "\n\ntemplate\nclass BatchingService\n{\n public:\n using PrepareFn =\n std::function>(\n ::grpc::ClientContext*, ::grpc::CompletionQueue*)>;\n using Callback = std::function;\n\n struct MessageType\n {\n Request& request;\n Response& response;\n Callback callback;\n };\n\n class Resources\n {\n public:\n Resources(PrepareFn prepare_fn, std::shared_ptr executor,\n std::shared_ptr<::trtlab::ThreadPool> post_process uint32_t max_batch_size,\n uint64_t timeout_in_us)\n : m_PrepareFn(prepare_fn), m_Executor(executor), m_WaitAndDone(post_process),\n m_MaxBatchSize(max_batch_size), m_Timeout(timeout_in_us)\n {\n }\n\n std::shared_ptr>\n CreateClient(std::function on_recv)\n {\n auto on_sent = [](Request&& request) {};\n\n return std::make_shared>(\n m_PrepareFn, m_Executor, on_sent, on_recv);\n }\n\n void Enqueue(Request& req, Response& resp, Callback callback)\n {\n m_MessageQueue.enqueue(MessageType{req, resp, callback});\n }\n\n protected:\n void BatchingEngine()\n {\n constexpr uint64_t quanta = 100;\n const double timeout = static_cast(m_Timeout - quanta) / 1000000.0;\n const size_t max_batch_size = m_MaxBatchSize;\n size_t total_count;\n size_t max_deque;\n std::shared_ptr> messages;\n std::chrono::time_point start;\n thread_local ConsumerToken token(m_MessageQueue);\n\n // clang-format off\n auto elapsed_time =\n [](std::chrono::time_point& start) -> double {\n return std::chrono::duration(\n std::chrono::high_resolution_clock::now() - start).count();\n };\n // clang-format on\n\n for(;;)\n {\n MessageType messages[m_MaxBatchsize];\n max_batch = m_MaxBatchsize;\n total_count = 0;\n\n // pull 1 element from the queue and start timer\n // if dequeue times outs, then restart the loop\n total_count = m_MessageQueue.wait_dequeue_bulk_timed(\n token, &(*messages)[total_count], 1, quanta);\n max_deque = max_batch_size - total_count;\n\n if(count == 0)\n {\n continue;\n }\n\n // Create a Corked Stream - Corked = buffered writes\n stream = CreateClient([messages](Response&& response) mutable {\n CHECK(!messges.empty());\n DLOG(INFO) << \"Finishing Unary Response/Callback: \" << message.size()\n << \" remain on queue\";\n auto m = messages.front();\n m.response = std::move(response);\n m.callback();\n messages.erase(messages.begin());\n });\n stream->Corked(true);\n\n // Continue to collect inference requests until we reach a maximum batch size\n // or we hit the timeout. We will eagerly forward our current batch items along\n // the stream so the preprocessor can get ahead start\n start = std::chrono::high_resolution_clock::now();\n while(total_count < max_batch_size && elapsed(start) <)\n {\n total_count += total_count = m_MessageQueue.wait_dequeue_bulk_timed(\n token, &(*messages)[total_count], max_deque, quanta);\n max_deque = max_batch_size - total_count;\n\n for(; isend < total_count; isend++)\n {\n auto& m = (*messages)[isend];\n stream->Write(m.request));\n }\n }\n\n // Batching complete\n if(total_count)\n {\n messages->resize(total_count);\n stream->Done();\n m_WaitAndFinish.enqueue([stream]() mutable {\n auto future = stream->Status();\n future.wait();\n streawm.reset();\n });\n messages.reset(new std::vector);\n messages->resize(max_batch_size);\n }\n }\n }\n\n private:\n PrepareFn m_PrepareFn;\n std::shared_ptr m_Executor;\n std::shared_ptr<::trtlab::ThreadPool> m_WaitAndDelete;\n size_t m_MaxBatchsize;\n uint64_t m_Timeout;\n BlockingConcurrentQueue m_MessageQueue;\n };\n\n class BatchingContext : public Context\n {\n void ExecuteRPC(Request& request, Response& response) final override\n {\n LOG(INFO) << \"incoming unary request\";\n this->GetResources()->Enqueue(&request, &response, [this](bool ok) {\n if(ok)\n {\n this->FinishResponse();\n }\n else\n {\n LOG(ERROR) << \"Upstream Error\";\n this->CancelResponse();\n }\n });\n }\n };\n};\n" }, { "path": "examples/ONNX/resnet50/README.md", "content": "# TensorRT ResNet50 Example\n\n- `fetch.sh` downloads the onnx model, test data, and calibration images from S3\n - after running this script the `resnet50` and `calibration_images` directories should be present in your\n local path\n\n- Build (`build.py`) TensorRT engines from the `model.onnx` file\n - cli options:\n - `--batch` will select the batch size, multiple can be given, a separate\n engine for each batch size will be generated.\n - `--precision` can be `fp32`, `fp16`, or `int8`. if multiple precision are given, an\n engine for each will be created. \n - **Note**: To use `int8` precision, you will need a Turing, Volta, or Pascal GPU with compute capability 6.1.\n - If you have a Turing or Volta GPU, then run the following commmand which will generates 4 engines:\n ```\n ./build.py --batch=1 --batch=8 --precision=fp16 --precision=int8 resnet50/model.onnx\n ```\n - If you have a Pascal GPU, run the following which generates 2 engines:\n ```\n ./build.py --batch=1 --batch=8 --precision=fp32 resnet50/model.onnx\n ```\n- Functional Test\n - `./run_onnx_tests.py model-b1-fp16.engine` will run the onnx tests \n\n- Benchmark TensorRT engines at different batch sizes and concurrent executions:\n - `/work/build/examples/00_TensorRT/infer.x --engine=model-b1-fp16.engine --contexts=1`\n - `/work/build/examples/00_TensorRT/infer.x --engine=model-b1-fp16.engine --contexts=8`\n - `/work/build/examples/00_TensorRT/infer.x --engine=model-b8-fp16.engine --contexts=1`\n - `/work/build/examples/00_TensorRT/infer.x --engine=model-b8-fp16.engine --contexts=6`\n\n- `./run_jpeg_test.py --image=images/broccoli-3784.jpg model-b1-fp16.engine`\n - Note this example requires MxNet for image preprocessing: `pip install mxnet`\n - On a V100 using FP16, your results should be close to\n ```\n *** Results ***\n broccoli 0.9511453\n ```\n- `./run_jpeg_test.py --image=images/broccoli-3784.jpg model-b1-int8.engine`\n - When using INT8, your results should be close to\n ```\n *** Results ***\n broccoli 0.9228073\n ```\n\n## Credits\n\n - [broccoli image](https://www.openfotos.com/view/broccoli-3784) - OpenFotos\n - https://www.openfotos.com/pages/open-fotos-license\n - [calibration images](calibration_images.csv) - Images from OpenFotos and Pixabay\n - https://www.openfotos.com/pages/open-fotos-license\n - https://pixabay.com/service/license/ \n\n## TODOs\n\n - [ ] Update `run_jpeg_test.py` to highlight the async interface.\n" }, { "path": "examples/ONNX/resnet50/build.py", "content": "#!/usr/bin/env python3\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\nimport os\nimport subprocess\n\nimport click\nimport int8\n\nprecision_opts = {\n \"fp32\": \"\",\n \"fp16\": \"--fp16\",\n \"int8\": \"--fp16 --int8\",\n}\n\nFile = click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True)\n\n@click.command()\n@click.option(\"--batch\", type=click.IntRange(min=1, max=32), multiple=True)\n@click.option(\"--precision\", type=click.Choice([\"fp32\", \"fp16\", \"int8\"]), multiple=True)\n@click.argument(\"models\", type=File, nargs=-1)\ndef main(models, batch, precision):\n for model in models:\n #click.echo(model)\n #click.echo(precision)\n for p in precision:\n #click.echo(p)\n for b in batch:\n #click.echo(b)\n n = \"b{}-{}\".format(b, p)\n m = os.path.basename(model)\n m, ext = os.path.splitext(m)\n e = \"{}-{}.{}\".format(m,n,\"engine\")\n\n if os.path.isfile(e):\n print(\"A TensorRT engine {} already exists! Skipping...\".format(e))\n continue\n elif p == \"int8\":\n assert os.path.isdir(\"./calibration_images\"), \"Need to download calibration images before creating INT8 engine!\"\n int8.build_int8_engine_onnx(model, \"./calibration_images\", b, 32, e)\n else:\n subprocess.call(\"trtexec --onnx={} --batch={} {} --saveEngine={}\".format(model, b, precision_opts.get(p), e), shell=True)\n\nif __name__ == \"__main__\":\n main()\n\n" }, { "path": "examples/ONNX/resnet50/calibration_images.csv", "content": "url,license,label\nhttps://www.openfotos.com/pictures/red-rock-crab-1096.full.jpg,Open Fotos License,crab\nhttps://www.openfotos.com/pictures/lazy-cats-4109.full.jpg,Open Fotos License,cat\nhttps://www.openfotos.com/pictures/beautiful-elephants-4005.full.jpg,Open Fotos License,elephants\nhttps://www.openfotos.com/pictures/funny-wild-pigs-437.full.jpg,Open Fotos License,pig\nhttps://www.openfotos.com/pictures/hopping-5017.full.jpg,Open Fotos License,bird\nhttps://www.openfotos.com/pictures/old-car-4811.full.jpg,Open Fotos License,car\nhttps://www.openfotos.com/pictures/audi-4740.full.jpg,Open Fotos License,car\nhttps://www.openfotos.com/pictures/boarding-in-the-plane-4891.full.jpg,Open Fotos License,plane\nhttps://cdn.pixabay.com/photo/2016/03/09/09/28/bear-1245807_960_720.jpg,Pixabay License,bear\nhttps://cdn.pixabay.com/photo/2015/02/26/06/09/panda-649938_960_720.jpg,Pixabay License,giant panda\nhttps://cdn.pixabay.com/photo/2015/09/22/19/00/ship-952292_960_720.jpg,Pixabay License,boat\nhttps://cdn.pixabay.com/photo/2017/01/16/19/17/horses-1984977_960_720.jpg,Pixabay License,horse\nhttps://cdn.pixabay.com/photo/2015/03/26/09/54/pug-690566_960_720.jpg,Pixabay License,dog\nhttps://cdn.pixabay.com/photo/2016/08/19/15/23/lizard-1605515_960_720.jpg,Pixabay License,lizzard\nhttps://cdn.pixabay.com/photo/2016/07/09/12/16/apple-1506119_960_720.jpg,Pixabay License,apple\nhttps://cdn.pixabay.com/photo/2016/10/07/14/11/tangerines-1721633_960_720.jpg,Pixabay License,orange\nhttps://cdn.pixabay.com/photo/2015/06/19/16/48/watermelon-815072_960_720.jpg,Pixabay License,watermelon\nhttps://cdn.pixabay.com/photo/2017/06/09/16/39/carrots-2387394_960_720.jpg,Pixabay License,carrot\nhttps://cdn.pixabay.com/photo/2019/02/28/22/45/hippo-4027011_960_720.jpg,Pixabay License,hippo\nhttps://cdn.pixabay.com/photo/2012/03/04/00/09/africa-21787_960_720.jpg,Pixabay License,lion\nhttps://cdn.pixabay.com/photo/2015/07/27/19/47/turtle-863336_960_720.jpg,Pixabay License,turtle\nhttps://cdn.pixabay.com/photo/2018/04/15/17/45/fish-3322230_960_720.jpg,Pixabay License,fish\nhttps://cdn.pixabay.com/photo/2013/11/01/11/13/dolphin-203875_960_720.jpg,Pixabay License,dolphin\nhttps://cdn.pixabay.com/photo/2014/05/02/21/49/home-office-336373_960_720.jpg,Pixabay License,laptop\nhttps://cdn.pixabay.com/photo/2012/05/18/21/45/tiger-mosquito-49141_960_720.jpg,Pixabay License,mosquito\nhttps://cdn.pixabay.com/photo/2016/08/09/13/21/coffee-1580595_960_720.jpg,Pixabay License,cup\nhttps://cdn.pixabay.com/photo/2016/02/19/10/36/lemons-1209309_960_720.jpg,Pixabay License,lemon\nhttps://cdn.pixabay.com/photo/2018/12/14/02/41/chengdu-3874136_960_720.jpg,Pixabay License,hotpot\nhttps://cdn.pixabay.com/photo/2015/09/05/12/53/violin-924349_960_720.jpg,Pixabay License,violin\nhttps://cdn.pixabay.com/photo/2015/11/07/11/22/pillows-1031079_960_720.jpg,Pixabay License,pillow\nhttps://cdn.pixabay.com/photo/2016/11/18/22/26/animal-1837164_960_720.jpg,Pixabay License,rhino\nhttps://cdn.pixabay.com/photo/2016/12/04/21/58/rabbit-1882699_960_720.jpg,Pixabay License,rabbit\nhttps://cdn.pixabay.com/photo/2014/10/23/18/56/tiger-500118_960_720.jpg,Pixabay License,tiger\n" }, { "path": "examples/ONNX/resnet50/calibrator.py", "content": "#\n# Copyright 1993-2019 NVIDIA Corporation. All rights reserved.\n#\n# NOTICE TO LICENSEE:\n#\n# This source code and/or documentation (\"Licensed Deliverables\") are\n# subject to NVIDIA intellectual property rights under U.S. and\n# international Copyright laws.\n#\n# These Licensed Deliverables contained herein is PROPRIETARY and\n# CONFIDENTIAL to NVIDIA and is being provided under the terms and\n# conditions of a form of NVIDIA software license agreement by and\n# between NVIDIA and Licensee (\"License Agreement\") or electronically\n# accepted by Licensee. Notwithstanding any terms or conditions to\n# the contrary in the License Agreement, reproduction or disclosure\n# of the Licensed Deliverables to any third party without the express\n# written consent of NVIDIA is prohibited.\n#\n# NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE\n# LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE\n# SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS\n# PROVIDED \"AS IS\" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.\n# NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED\n# DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,\n# NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.\n# NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE\n# LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY\n# SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY\n# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,\n# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS\n# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE\n# OF THESE LICENSED DELIVERABLES.\n#\n# U.S. Government End Users. These Licensed Deliverables are a\n# \"commercial item\" as that term is defined at 48 C.F.R. 2.101 (OCT\n# 1995), consisting of \"commercial computer software\" and \"commercial\n# computer software documentation\" as such terms are used in 48\n# C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government\n# only as a commercial end item. Consistent with 48 C.F.R.12.212 and\n# 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all\n# U.S. Government End Users acquire the Licensed Deliverables with\n# only those rights set forth herein.\n#\n# Any use of the Licensed Deliverables in individual and commercial\n# software must include, in the user documentation and internal\n# comments to the code, the above Disclaimer and U.S. Government End\n# Users Notice.\n\nimport tensorrt as trt\nimport os\n\nimport pycuda.driver as cuda\nimport pycuda.autoinit\nimport matplotlib.pyplot as plt\n\nimport mxnet as mx\nfrom mxnet.gluon.data.vision import transforms\nimport numpy as np\nfrom random import shuffle\n\nclass ONNXEntropyCalibrator(trt.IInt8EntropyCalibrator):\n def __init__(self, image_dir, batch_size, calibration_batches, cache_file):\n # Whenever you specify a custom constructor for a TensorRT class,\n # you MUST call the constructor of the parent explicitly.\n trt.IInt8EntropyCalibrator.__init__(self)\n\n self.cache_file = cache_file\n\n # Get a list of all the images in the image directory.\n image_files = [os.path.join(image_dir, f) for f in os.listdir(image_dir)]\n shuffle(image_files)\n\n if len(image_files) < calibration_batches * batch_size:\n print(\"Only found enough images for {} batches instead of {}, continuing anyway...\".format(len(image_files) // batch_size, calibration_batches))\n self.image_files = image_files\n else:\n self.image_files = image_files[:calibration_batches * batch_size]\n\n # Keeps track of current image in image list\n self.current_image = 0\n self.batch_size = batch_size\n self.input_size = [3,224,224]\n\n # Each element of the calibration data is a float32.\n self.device_input = cuda.mem_alloc(self.batch_size * self.input_size[0] * self.input_size[1] * self.input_size[2] * trt.float32.itemsize)\n\n # Create a generator that will give us batches. We can use next() to iterate over the result.\n def load_batches():\n while self.current_image < len(self.image_files):\n data, images_read = self.read_image_batch()\n self.current_image += images_read\n yield data\n self.batches = load_batches()\n\n\n def transform_image(self, img):\n transform_fn = transforms.Compose([\n transforms.Resize(224),\n transforms.CenterCrop(224),\n transforms.ToTensor(),\n transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n ])\n img = transform_fn(mx.nd.array(img)).asnumpy()\n return img\n\n\n # This function is used to load calibration images into batches.\n def read_image_batch(self):\n # Depending on batch size and number of images, the final batch might only be partially full.\n images_to_read = min(self.batch_size, len(self.image_files) - self.current_image)\n\n host_buffer = np.zeros(shape=[self.batch_size]+self.input_size)\n for i in range(images_to_read):\n img = np.array(plt.imread(self.image_files[self.current_image]))\n img = self.transform_image(img)\n host_buffer[i,:,:,:] = img\n\n return host_buffer, images_to_read\n\n def get_batch_size(self):\n return self.batch_size\n\n # TensorRT passes along the names of the engine bindings to the get_batch function.\n # You don't necessarily have to use them, but they can be useful to understand the order of\n # the inputs. The bindings list is expected to have the same ordering as 'names'.\n def get_batch(self, names):\n try:\n # Get a single batch.\n data = np.ascontiguousarray(next(self.batches), np.float32)\n # Copy to device, then return a list containing pointers to input device buffers.\n cuda.memcpy_htod(self.device_input, data)\n return [int(self.device_input)]\n except StopIteration:\n # When we're out of batches, we return either [] or None.\n # This signals to TensorRT that there is no calibration data remaining.\n return None\n\n def read_calibration_cache(self):\n # If there is a cache, use it instead of calibrating again. Otherwise, implicitly return None.\n if os.path.exists(self.cache_file):\n with open(self.cache_file, \"rb\") as f:\n return f.read()\n\n def write_calibration_cache(self, cache):\n with open(self.cache_file, \"wb\") as f:\n f.write(cache)\n" }, { "path": "examples/ONNX/resnet50/fetch.sh", "content": "#!/bin/bash\n\nif [ ! -e \"resnet50.tar.gz\" ]; then\n wget https://s3.amazonaws.com/download.onnx/models/opset_8/resnet50.tar.gz\nfi\n\nif [ ! -e \"open_source_images.tar.gz\" ]; then\n wget https://s3-us-west-2.amazonaws.com/com.nvidia.tensorrt-laboratory/open_source_images.tar.gz\nfi\n\nif md5sum -c resnet50.md5; then\n if [ ! -e \"resnet50\" ]; then\n tar xzf resnet50.tar.gz\n fi\n echo \"ResNet50 download good\"\nelse\n echo \"ResNet50 md5 checksum failed\"\n exit 911\nfi\n\nif md5sum -c open_source_images.md5; then\n if [ ! -e \"calibration_images\" ]; then\n tar xf open_source_images.tar.gz\n fi\n echo \"All good - Continue to Build Phase\"\nelse\n echo \"calibration_images md5 checksum failed\"\n exit 911\nfi\n" }, { "path": "examples/ONNX/resnet50/imagenet_labels.py", "content": "\nlabels = {0: 'tench, Tinca tinca',\n 1: 'goldfish, Carassius auratus',\n 2: 'great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias',\n 3: 'tiger shark, Galeocerdo cuvieri',\n 4: 'hammerhead, hammerhead shark',\n 5: 'electric ray, crampfish, numbfish, torpedo',\n 6: 'stingray',\n 7: 'cock',\n 8: 'hen',\n 9: 'ostrich, Struthio camelus',\n 10: 'brambling, Fringilla montifringilla',\n 11: 'goldfinch, Carduelis carduelis',\n 12: 'house finch, linnet, Carpodacus mexicanus',\n 13: 'junco, snowbird',\n 14: 'indigo bunting, indigo finch, indigo bird, Passerina cyanea',\n 15: 'robin, American robin, Turdus migratorius',\n 16: 'bulbul',\n 17: 'jay',\n 18: 'magpie',\n 19: 'chickadee',\n 20: 'water ouzel, dipper',\n 21: 'kite',\n 22: 'bald eagle, American eagle, Haliaeetus leucocephalus',\n 23: 'vulture',\n 24: 'great grey owl, great gray owl, Strix nebulosa',\n 25: 'European fire salamander, Salamandra salamandra',\n 26: 'common newt, Triturus vulgaris',\n 27: 'eft',\n 28: 'spotted salamander, Ambystoma maculatum',\n 29: 'axolotl, mud puppy, Ambystoma mexicanum',\n 30: 'bullfrog, Rana catesbeiana',\n 31: 'tree frog, tree-frog',\n 32: 'tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui',\n 33: 'loggerhead, loggerhead turtle, Caretta caretta',\n 34: 'leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea',\n 35: 'mud turtle',\n 36: 'terrapin',\n 37: 'box turtle, box tortoise',\n 38: 'banded gecko',\n 39: 'common iguana, iguana, Iguana iguana',\n 40: 'American chameleon, anole, Anolis carolinensis',\n 41: 'whiptail, whiptail lizard',\n 42: 'agama',\n 43: 'frilled lizard, Chlamydosaurus kingi',\n 44: 'alligator lizard',\n 45: 'Gila monster, Heloderma suspectum',\n 46: 'green lizard, Lacerta viridis',\n 47: 'African chameleon, Chamaeleo chamaeleon',\n 48: 'Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis',\n 49: 'African crocodile, Nile crocodile, Crocodylus niloticus',\n 50: 'American alligator, Alligator mississipiensis',\n 51: 'triceratops',\n 52: 'thunder snake, worm snake, Carphophis amoenus',\n 53: 'ringneck snake, ring-necked snake, ring snake',\n 54: 'hognose snake, puff adder, sand viper',\n 55: 'green snake, grass snake',\n 56: 'king snake, kingsnake',\n 57: 'garter snake, grass snake',\n 58: 'water snake',\n 59: 'vine snake',\n 60: 'night snake, Hypsiglena torquata',\n 61: 'boa constrictor, Constrictor constrictor',\n 62: 'rock python, rock snake, Python sebae',\n 63: 'Indian cobra, Naja naja',\n 64: 'green mamba',\n 65: 'sea snake',\n 66: 'horned viper, cerastes, sand viper, horned asp, Cerastes cornutus',\n 67: 'diamondback, diamondback rattlesnake, Crotalus adamanteus',\n 68: 'sidewinder, horned rattlesnake, Crotalus cerastes',\n 69: 'trilobite',\n 70: 'harvestman, daddy longlegs, Phalangium opilio',\n 71: 'scorpion',\n 72: 'black and gold garden spider, Argiope aurantia',\n 73: 'barn spider, Araneus cavaticus',\n 74: 'garden spider, Aranea diademata',\n 75: 'black widow, Latrodectus mactans',\n 76: 'tarantula',\n 77: 'wolf spider, hunting spider',\n 78: 'tick',\n 79: 'centipede',\n 80: 'black grouse',\n 81: 'ptarmigan',\n 82: 'ruffed grouse, partridge, Bonasa umbellus',\n 83: 'prairie chicken, prairie grouse, prairie fowl',\n 84: 'peacock',\n 85: 'quail',\n 86: 'partridge',\n 87: 'African grey, African gray, Psittacus erithacus',\n 88: 'macaw',\n 89: 'sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita',\n 90: 'lorikeet',\n 91: 'coucal',\n 92: 'bee eater',\n 93: 'hornbill',\n 94: 'hummingbird',\n 95: 'jacamar',\n 96: 'toucan',\n 97: 'drake',\n 98: 'red-breasted merganser, Mergus serrator',\n 99: 'goose',\n 100: 'black swan, Cygnus atratus',\n 101: 'tusker',\n 102: 'echidna, spiny anteater, anteater',\n 103: 'platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus',\n 104: 'wallaby, brush kangaroo',\n 105: 'koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus',\n 106: 'wombat',\n 107: 'jellyfish',\n 108: 'sea anemone, anemone',\n 109: 'brain coral',\n 110: 'flatworm, platyhelminth',\n 111: 'nematode, nematode worm, roundworm',\n 112: 'conch',\n 113: 'snail',\n 114: 'slug',\n 115: 'sea slug, nudibranch',\n 116: 'chiton, coat-of-mail shell, sea cradle, polyplacophore',\n 117: 'chambered nautilus, pearly nautilus, nautilus',\n 118: 'Dungeness crab, Cancer magister',\n 119: 'rock crab, Cancer irroratus',\n 120: 'fiddler crab',\n 121: 'king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica',\n 122: 'American lobster, Northern lobster, Maine lobster, Homarus americanus',\n 123: 'spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish',\n 124: 'crayfish, crawfish, crawdad, crawdaddy',\n 125: 'hermit crab',\n 126: 'isopod',\n 127: 'white stork, Ciconia ciconia',\n 128: 'black stork, Ciconia nigra',\n 129: 'spoonbill',\n 130: 'flamingo',\n 131: 'little blue heron, Egretta caerulea',\n 132: 'American egret, great white heron, Egretta albus',\n 133: 'bittern',\n 134: 'crane',\n 135: 'limpkin, Aramus pictus',\n 136: 'European gallinule, Porphyrio porphyrio',\n 137: 'American coot, marsh hen, mud hen, water hen, Fulica americana',\n 138: 'bustard',\n 139: 'ruddy turnstone, Arenaria interpres',\n 140: 'red-backed sandpiper, dunlin, Erolia alpina',\n 141: 'redshank, Tringa totanus',\n 142: 'dowitcher',\n 143: 'oystercatcher, oyster catcher',\n 144: 'pelican',\n 145: 'king penguin, Aptenodytes patagonica',\n 146: 'albatross, mollymawk',\n 147: 'grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus',\n 148: 'killer whale, killer, orca, grampus, sea wolf, Orcinus orca',\n 149: 'dugong, Dugong dugon',\n 150: 'sea lion',\n 151: 'Chihuahua',\n 152: 'Japanese spaniel',\n 153: 'Maltese dog, Maltese terrier, Maltese',\n 154: 'Pekinese, Pekingese, Peke',\n 155: 'Shih-Tzu',\n 156: 'Blenheim spaniel',\n 157: 'papillon',\n 158: 'toy terrier',\n 159: 'Rhodesian ridgeback',\n 160: 'Afghan hound, Afghan',\n 161: 'basset, basset hound',\n 162: 'beagle',\n 163: 'bloodhound, sleuthhound',\n 164: 'bluetick',\n 165: 'black-and-tan coonhound',\n 166: 'Walker hound, Walker foxhound',\n 167: 'English foxhound',\n 168: 'redbone',\n 169: 'borzoi, Russian wolfhound',\n 170: 'Irish wolfhound',\n 171: 'Italian greyhound',\n 172: 'whippet',\n 173: 'Ibizan hound, Ibizan Podenco',\n 174: 'Norwegian elkhound, elkhound',\n 175: 'otterhound, otter hound',\n 176: 'Saluki, gazelle hound',\n 177: 'Scottish deerhound, deerhound',\n 178: 'Weimaraner',\n 179: 'Staffordshire bullterrier, Staffordshire bull terrier',\n 180: 'American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier',\n 181: 'Bedlington terrier',\n 182: 'Border terrier',\n 183: 'Kerry blue terrier',\n 184: 'Irish terrier',\n 185: 'Norfolk terrier',\n 186: 'Norwich terrier',\n 187: 'Yorkshire terrier',\n 188: 'wire-haired fox terrier',\n 189: 'Lakeland terrier',\n 190: 'Sealyham terrier, Sealyham',\n 191: 'Airedale, Airedale terrier',\n 192: 'cairn, cairn terrier',\n 193: 'Australian terrier',\n 194: 'Dandie Dinmont, Dandie Dinmont terrier',\n 195: 'Boston bull, Boston terrier',\n 196: 'miniature schnauzer',\n 197: 'giant schnauzer',\n 198: 'standard schnauzer',\n 199: 'Scotch terrier, Scottish terrier, Scottie',\n 200: 'Tibetan terrier, chrysanthemum dog',\n 201: 'silky terrier, Sydney silky',\n 202: 'soft-coated wheaten terrier',\n 203: 'West Highland white terrier',\n 204: 'Lhasa, Lhasa apso',\n 205: 'flat-coated retriever',\n 206: 'curly-coated retriever',\n 207: 'golden retriever',\n 208: 'Labrador retriever',\n 209: 'Chesapeake Bay retriever',\n 210: 'German short-haired pointer',\n 211: 'vizsla, Hungarian pointer',\n 212: 'English setter',\n 213: 'Irish setter, red setter',\n 214: 'Gordon setter',\n 215: 'Brittany spaniel',\n 216: 'clumber, clumber spaniel',\n 217: 'English springer, English springer spaniel',\n 218: 'Welsh springer spaniel',\n 219: 'cocker spaniel, English cocker spaniel, cocker',\n 220: 'Sussex spaniel',\n 221: 'Irish water spaniel',\n 222: 'kuvasz',\n 223: 'schipperke',\n 224: 'groenendael',\n 225: 'malinois',\n 226: 'briard',\n 227: 'kelpie',\n 228: 'komondor',\n 229: 'Old English sheepdog, bobtail',\n 230: 'Shetland sheepdog, Shetland sheep dog, Shetland',\n 231: 'collie',\n 232: 'Border collie',\n 233: 'Bouvier des Flandres, Bouviers des Flandres',\n 234: 'Rottweiler',\n 235: 'German shepherd, German shepherd dog, German police dog, alsatian',\n 236: 'Doberman, Doberman pinscher',\n 237: 'miniature pinscher',\n 238: 'Greater Swiss Mountain dog',\n 239: 'Bernese mountain dog',\n 240: 'Appenzeller',\n 241: 'EntleBucher',\n 242: 'boxer',\n 243: 'bull mastiff',\n 244: 'Tibetan mastiff',\n 245: 'French bulldog',\n 246: 'Great Dane',\n 247: 'Saint Bernard, St Bernard',\n 248: 'Eskimo dog, husky',\n 249: 'malamute, malemute, Alaskan malamute',\n 250: 'Siberian husky',\n 251: 'dalmatian, coach dog, carriage dog',\n 252: 'affenpinscher, monkey pinscher, monkey dog',\n 253: 'basenji',\n 254: 'pug, pug-dog',\n 255: 'Leonberg',\n 256: 'Newfoundland, Newfoundland dog',\n 257: 'Great Pyrenees',\n 258: 'Samoyed, Samoyede',\n 259: 'Pomeranian',\n 260: 'chow, chow chow',\n 261: 'keeshond',\n 262: 'Brabancon griffon',\n 263: 'Pembroke, Pembroke Welsh corgi',\n 264: 'Cardigan, Cardigan Welsh corgi',\n 265: 'toy poodle',\n 266: 'miniature poodle',\n 267: 'standard poodle',\n 268: 'Mexican hairless',\n 269: 'timber wolf, grey wolf, gray wolf, Canis lupus',\n 270: 'white wolf, Arctic wolf, Canis lupus tundrarum',\n 271: 'red wolf, maned wolf, Canis rufus, Canis niger',\n 272: 'coyote, prairie wolf, brush wolf, Canis latrans',\n 273: 'dingo, warrigal, warragal, Canis dingo',\n 274: 'dhole, Cuon alpinus',\n 275: 'African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus',\n 276: 'hyena, hyaena',\n 277: 'red fox, Vulpes vulpes',\n 278: 'kit fox, Vulpes macrotis',\n 279: 'Arctic fox, white fox, Alopex lagopus',\n 280: 'grey fox, gray fox, Urocyon cinereoargenteus',\n 281: 'tabby, tabby cat',\n 282: 'tiger cat',\n 283: 'Persian cat',\n 284: 'Siamese cat, Siamese',\n 285: 'Egyptian cat',\n 286: 'cougar, puma, catamount, mountain lion, painter, panther, Felis concolor',\n 287: 'lynx, catamount',\n 288: 'leopard, Panthera pardus',\n 289: 'snow leopard, ounce, Panthera uncia',\n 290: 'jaguar, panther, Panthera onca, Felis onca',\n 291: 'lion, king of beasts, Panthera leo',\n 292: 'tiger, Panthera tigris',\n 293: 'cheetah, chetah, Acinonyx jubatus',\n 294: 'brown bear, bruin, Ursus arctos',\n 295: 'American black bear, black bear, Ursus americanus, Euarctos americanus',\n 296: 'ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus',\n 297: 'sloth bear, Melursus ursinus, Ursus ursinus',\n 298: 'mongoose',\n 299: 'meerkat, mierkat',\n 300: 'tiger beetle',\n 301: 'ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle',\n 302: 'ground beetle, carabid beetle',\n 303: 'long-horned beetle, longicorn, longicorn beetle',\n 304: 'leaf beetle, chrysomelid',\n 305: 'dung beetle',\n 306: 'rhinoceros beetle',\n 307: 'weevil',\n 308: 'fly',\n 309: 'bee',\n 310: 'ant, emmet, pismire',\n 311: 'grasshopper, hopper',\n 312: 'cricket',\n 313: 'walking stick, walkingstick, stick insect',\n 314: 'cockroach, roach',\n 315: 'mantis, mantid',\n 316: 'cicada, cicala',\n 317: 'leafhopper',\n 318: 'lacewing, lacewing fly',\n 319: \"dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk\",\n 320: 'damselfly',\n 321: 'admiral',\n 322: 'ringlet, ringlet butterfly',\n 323: 'monarch, monarch butterfly, milkweed butterfly, Danaus plexippus',\n 324: 'cabbage butterfly',\n 325: 'sulphur butterfly, sulfur butterfly',\n 326: 'lycaenid, lycaenid butterfly',\n 327: 'starfish, sea star',\n 328: 'sea urchin',\n 329: 'sea cucumber, holothurian',\n 330: 'wood rabbit, cottontail, cottontail rabbit',\n 331: 'hare',\n 332: 'Angora, Angora rabbit',\n 333: 'hamster',\n 334: 'porcupine, hedgehog',\n 335: 'fox squirrel, eastern fox squirrel, Sciurus niger',\n 336: 'marmot',\n 337: 'beaver',\n 338: 'guinea pig, Cavia cobaya',\n 339: 'sorrel',\n 340: 'zebra',\n 341: 'hog, pig, grunter, squealer, Sus scrofa',\n 342: 'wild boar, boar, Sus scrofa',\n 343: 'warthog',\n 344: 'hippopotamus, hippo, river horse, Hippopotamus amphibius',\n 345: 'ox',\n 346: 'water buffalo, water ox, Asiatic buffalo, Bubalus bubalis',\n 347: 'bison',\n 348: 'ram, tup',\n 349: 'bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis',\n 350: 'ibex, Capra ibex',\n 351: 'hartebeest',\n 352: 'impala, Aepyceros melampus',\n 353: 'gazelle',\n 354: 'Arabian camel, dromedary, Camelus dromedarius',\n 355: 'llama',\n 356: 'weasel',\n 357: 'mink',\n 358: 'polecat, fitch, foulmart, foumart, Mustela putorius',\n 359: 'black-footed ferret, ferret, Mustela nigripes',\n 360: 'otter',\n 361: 'skunk, polecat, wood pussy',\n 362: 'badger',\n 363: 'armadillo',\n 364: 'three-toed sloth, ai, Bradypus tridactylus',\n 365: 'orangutan, orang, orangutang, Pongo pygmaeus',\n 366: 'gorilla, Gorilla gorilla',\n 367: 'chimpanzee, chimp, Pan troglodytes',\n 368: 'gibbon, Hylobates lar',\n 369: 'siamang, Hylobates syndactylus, Symphalangus syndactylus',\n 370: 'guenon, guenon monkey',\n 371: 'patas, hussar monkey, Erythrocebus patas',\n 372: 'baboon',\n 373: 'macaque',\n 374: 'langur',\n 375: 'colobus, colobus monkey',\n 376: 'proboscis monkey, Nasalis larvatus',\n 377: 'marmoset',\n 378: 'capuchin, ringtail, Cebus capucinus',\n 379: 'howler monkey, howler',\n 380: 'titi, titi monkey',\n 381: 'spider monkey, Ateles geoffroyi',\n 382: 'squirrel monkey, Saimiri sciureus',\n 383: 'Madagascar cat, ring-tailed lemur, Lemur catta',\n 384: 'indri, indris, Indri indri, Indri brevicaudatus',\n 385: 'Indian elephant, Elephas maximus',\n 386: 'African elephant, Loxodonta africana',\n 387: 'lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens',\n 388: 'giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca',\n 389: 'barracouta, snoek',\n 390: 'eel',\n 391: 'coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch',\n 392: 'rock beauty, Holocanthus tricolor',\n 393: 'anemone fish',\n 394: 'sturgeon',\n 395: 'gar, garfish, garpike, billfish, Lepisosteus osseus',\n 396: 'lionfish',\n 397: 'puffer, pufferfish, blowfish, globefish',\n 398: 'abacus',\n 399: 'abaya',\n 400: \"academic gown, academic robe, judge's robe\",\n 401: 'accordion, piano accordion, squeeze box',\n 402: 'acoustic guitar',\n 403: 'aircraft carrier, carrier, flattop, attack aircraft carrier',\n 404: 'airliner',\n 405: 'airship, dirigible',\n 406: 'altar',\n 407: 'ambulance',\n 408: 'amphibian, amphibious vehicle',\n 409: 'analog clock',\n 410: 'apiary, bee house',\n 411: 'apron',\n 412: 'ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin',\n 413: 'assault rifle, assault gun',\n 414: 'backpack, back pack, knapsack, packsack, rucksack, haversack',\n 415: 'bakery, bakeshop, bakehouse',\n 416: 'balance beam, beam',\n 417: 'balloon',\n 418: 'ballpoint, ballpoint pen, ballpen, Biro',\n 419: 'Band Aid',\n 420: 'banjo',\n 421: 'bannister, banister, balustrade, balusters, handrail',\n 422: 'barbell',\n 423: 'barber chair',\n 424: 'barbershop',\n 425: 'barn',\n 426: 'barometer',\n 427: 'barrel, cask',\n 428: 'barrow, garden cart, lawn cart, wheelbarrow',\n 429: 'baseball',\n 430: 'basketball',\n 431: 'bassinet',\n 432: 'bassoon',\n 433: 'bathing cap, swimming cap',\n 434: 'bath towel',\n 435: 'bathtub, bathing tub, bath, tub',\n 436: 'beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon',\n 437: 'beacon, lighthouse, beacon light, pharos',\n 438: 'beaker',\n 439: 'bearskin, busby, shako',\n 440: 'beer bottle',\n 441: 'beer glass',\n 442: 'bell cote, bell cot',\n 443: 'bib',\n 444: 'bicycle-built-for-two, tandem bicycle, tandem',\n 445: 'bikini, two-piece',\n 446: 'binder, ring-binder',\n 447: 'binoculars, field glasses, opera glasses',\n 448: 'birdhouse',\n 449: 'boathouse',\n 450: 'bobsled, bobsleigh, bob',\n 451: 'bolo tie, bolo, bola tie, bola',\n 452: 'bonnet, poke bonnet',\n 453: 'bookcase',\n 454: 'bookshop, bookstore, bookstall',\n 455: 'bottlecap',\n 456: 'bow',\n 457: 'bow tie, bow-tie, bowtie',\n 458: 'brass, memorial tablet, plaque',\n 459: 'brassiere, bra, bandeau',\n 460: 'breakwater, groin, groyne, mole, bulwark, seawall, jetty',\n 461: 'breastplate, aegis, egis',\n 462: 'broom',\n 463: 'bucket, pail',\n 464: 'buckle',\n 465: 'bulletproof vest',\n 466: 'bullet train, bullet',\n 467: 'butcher shop, meat market',\n 468: 'cab, hack, taxi, taxicab',\n 469: 'caldron, cauldron',\n 470: 'candle, taper, wax light',\n 471: 'cannon',\n 472: 'canoe',\n 473: 'can opener, tin opener',\n 474: 'cardigan',\n 475: 'car mirror',\n 476: 'carousel, carrousel, merry-go-round, roundabout, whirligig',\n 477: \"carpenter's kit, tool kit\",\n 478: 'carton',\n 479: 'car wheel',\n 480: 'cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM',\n 481: 'cassette',\n 482: 'cassette player',\n 483: 'castle',\n 484: 'catamaran',\n 485: 'CD player',\n 486: 'cello, violoncello',\n 487: 'cellular telephone, cellular phone, cellphone, cell, mobile phone',\n 488: 'chain',\n 489: 'chainlink fence',\n 490: 'chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour',\n 491: 'chain saw, chainsaw',\n 492: 'chest',\n 493: 'chiffonier, commode',\n 494: 'chime, bell, gong',\n 495: 'china cabinet, china closet',\n 496: 'Christmas stocking',\n 497: 'church, church building',\n 498: 'cinema, movie theater, movie theatre, movie house, picture palace',\n 499: 'cleaver, meat cleaver, chopper',\n 500: 'cliff dwelling',\n 501: 'cloak',\n 502: 'clog, geta, patten, sabot',\n 503: 'cocktail shaker',\n 504: 'coffee mug',\n 505: 'coffeepot',\n 506: 'coil, spiral, volute, whorl, helix',\n 507: 'combination lock',\n 508: 'computer keyboard, keypad',\n 509: 'confectionery, confectionary, candy store',\n 510: 'container ship, containership, container vessel',\n 511: 'convertible',\n 512: 'corkscrew, bottle screw',\n 513: 'cornet, horn, trumpet, trump',\n 514: 'cowboy boot',\n 515: 'cowboy hat, ten-gallon hat',\n 516: 'cradle',\n 517: 'crane',\n 518: 'crash helmet',\n 519: 'crate',\n 520: 'crib, cot',\n 521: 'Crock Pot',\n 522: 'croquet ball',\n 523: 'crutch',\n 524: 'cuirass',\n 525: 'dam, dike, dyke',\n 526: 'desk',\n 527: 'desktop computer',\n 528: 'dial telephone, dial phone',\n 529: 'diaper, nappy, napkin',\n 530: 'digital clock',\n 531: 'digital watch',\n 532: 'dining table, board',\n 533: 'dishrag, dishcloth',\n 534: 'dishwasher, dish washer, dishwashing machine',\n 535: 'disk brake, disc brake',\n 536: 'dock, dockage, docking facility',\n 537: 'dogsled, dog sled, dog sleigh',\n 538: 'dome',\n 539: 'doormat, welcome mat',\n 540: 'drilling platform, offshore rig',\n 541: 'drum, membranophone, tympan',\n 542: 'drumstick',\n 543: 'dumbbell',\n 544: 'Dutch oven',\n 545: 'electric fan, blower',\n 546: 'electric guitar',\n 547: 'electric locomotive',\n 548: 'entertainment center',\n 549: 'envelope',\n 550: 'espresso maker',\n 551: 'face powder',\n 552: 'feather boa, boa',\n 553: 'file, file cabinet, filing cabinet',\n 554: 'fireboat',\n 555: 'fire engine, fire truck',\n 556: 'fire screen, fireguard',\n 557: 'flagpole, flagstaff',\n 558: 'flute, transverse flute',\n 559: 'folding chair',\n 560: 'football helmet',\n 561: 'forklift',\n 562: 'fountain',\n 563: 'fountain pen',\n 564: 'four-poster',\n 565: 'freight car',\n 566: 'French horn, horn',\n 567: 'frying pan, frypan, skillet',\n 568: 'fur coat',\n 569: 'garbage truck, dustcart',\n 570: 'gasmask, respirator, gas helmet',\n 571: 'gas pump, gasoline pump, petrol pump, island dispenser',\n 572: 'goblet',\n 573: 'go-kart',\n 574: 'golf ball',\n 575: 'golfcart, golf cart',\n 576: 'gondola',\n 577: 'gong, tam-tam',\n 578: 'gown',\n 579: 'grand piano, grand',\n 580: 'greenhouse, nursery, glasshouse',\n 581: 'grille, radiator grille',\n 582: 'grocery store, grocery, food market, market',\n 583: 'guillotine',\n 584: 'hair slide',\n 585: 'hair spray',\n 586: 'half track',\n 587: 'hammer',\n 588: 'hamper',\n 589: 'hand blower, blow dryer, blow drier, hair dryer, hair drier',\n 590: 'hand-held computer, hand-held microcomputer',\n 591: 'handkerchief, hankie, hanky, hankey',\n 592: 'hard disc, hard disk, fixed disk',\n 593: 'harmonica, mouth organ, harp, mouth harp',\n 594: 'harp',\n 595: 'harvester, reaper',\n 596: 'hatchet',\n 597: 'holster',\n 598: 'home theater, home theatre',\n 599: 'honeycomb',\n 600: 'hook, claw',\n 601: 'hoopskirt, crinoline',\n 602: 'horizontal bar, high bar',\n 603: 'horse cart, horse-cart',\n 604: 'hourglass',\n 605: 'iPod',\n 606: 'iron, smoothing iron',\n 607: \"jack-o'-lantern\",\n 608: 'jean, blue jean, denim',\n 609: 'jeep, landrover',\n 610: 'jersey, T-shirt, tee shirt',\n 611: 'jigsaw puzzle',\n 612: 'jinrikisha, ricksha, rickshaw',\n 613: 'joystick',\n 614: 'kimono',\n 615: 'knee pad',\n 616: 'knot',\n 617: 'lab coat, laboratory coat',\n 618: 'ladle',\n 619: 'lampshade, lamp shade',\n 620: 'laptop, laptop computer',\n 621: 'lawn mower, mower',\n 622: 'lens cap, lens cover',\n 623: 'letter opener, paper knife, paperknife',\n 624: 'library',\n 625: 'lifeboat',\n 626: 'lighter, light, igniter, ignitor',\n 627: 'limousine, limo',\n 628: 'liner, ocean liner',\n 629: 'lipstick, lip rouge',\n 630: 'Loafer',\n 631: 'lotion',\n 632: 'loudspeaker, speaker, speaker unit, loudspeaker system, speaker system',\n 633: \"loupe, jeweler's loupe\",\n 634: 'lumbermill, sawmill',\n 635: 'magnetic compass',\n 636: 'mailbag, postbag',\n 637: 'mailbox, letter box',\n 638: 'maillot',\n 639: 'maillot, tank suit',\n 640: 'manhole cover',\n 641: 'maraca',\n 642: 'marimba, xylophone',\n 643: 'mask',\n 644: 'matchstick',\n 645: 'maypole',\n 646: 'maze, labyrinth',\n 647: 'measuring cup',\n 648: 'medicine chest, medicine cabinet',\n 649: 'megalith, megalithic structure',\n 650: 'microphone, mike',\n 651: 'microwave, microwave oven',\n 652: 'military uniform',\n 653: 'milk can',\n 654: 'minibus',\n 655: 'miniskirt, mini',\n 656: 'minivan',\n 657: 'missile',\n 658: 'mitten',\n 659: 'mixing bowl',\n 660: 'mobile home, manufactured home',\n 661: 'Model T',\n 662: 'modem',\n 663: 'monastery',\n 664: 'monitor',\n 665: 'moped',\n 666: 'mortar',\n 667: 'mortarboard',\n 668: 'mosque',\n 669: 'mosquito net',\n 670: 'motor scooter, scooter',\n 671: 'mountain bike, all-terrain bike, off-roader',\n 672: 'mountain tent',\n 673: 'mouse, computer mouse',\n 674: 'mousetrap',\n 675: 'moving van',\n 676: 'muzzle',\n 677: 'nail',\n 678: 'neck brace',\n 679: 'necklace',\n 680: 'nipple',\n 681: 'notebook, notebook computer',\n 682: 'obelisk',\n 683: 'oboe, hautboy, hautbois',\n 684: 'ocarina, sweet potato',\n 685: 'odometer, hodometer, mileometer, milometer',\n 686: 'oil filter',\n 687: 'organ, pipe organ',\n 688: 'oscilloscope, scope, cathode-ray oscilloscope, CRO',\n 689: 'overskirt',\n 690: 'oxcart',\n 691: 'oxygen mask',\n 692: 'packet',\n 693: 'paddle, boat paddle',\n 694: 'paddlewheel, paddle wheel',\n 695: 'padlock',\n 696: 'paintbrush',\n 697: \"pajama, pyjama, pj's, jammies\",\n 698: 'palace',\n 699: 'panpipe, pandean pipe, syrinx',\n 700: 'paper towel',\n 701: 'parachute, chute',\n 702: 'parallel bars, bars',\n 703: 'park bench',\n 704: 'parking meter',\n 705: 'passenger car, coach, carriage',\n 706: 'patio, terrace',\n 707: 'pay-phone, pay-station',\n 708: 'pedestal, plinth, footstall',\n 709: 'pencil box, pencil case',\n 710: 'pencil sharpener',\n 711: 'perfume, essence',\n 712: 'Petri dish',\n 713: 'photocopier',\n 714: 'pick, plectrum, plectron',\n 715: 'pickelhaube',\n 716: 'picket fence, paling',\n 717: 'pickup, pickup truck',\n 718: 'pier',\n 719: 'piggy bank, penny bank',\n 720: 'pill bottle',\n 721: 'pillow',\n 722: 'ping-pong ball',\n 723: 'pinwheel',\n 724: 'pirate, pirate ship',\n 725: 'pitcher, ewer',\n 726: \"plane, carpenter's plane, woodworking plane\",\n 727: 'planetarium',\n 728: 'plastic bag',\n 729: 'plate rack',\n 730: 'plow, plough',\n 731: \"plunger, plumber's helper\",\n 732: 'Polaroid camera, Polaroid Land camera',\n 733: 'pole',\n 734: 'police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria',\n 735: 'poncho',\n 736: 'pool table, billiard table, snooker table',\n 737: 'pop bottle, soda bottle',\n 738: 'pot, flowerpot',\n 739: \"potter's wheel\",\n 740: 'power drill',\n 741: 'prayer rug, prayer mat',\n 742: 'printer',\n 743: 'prison, prison house',\n 744: 'projectile, missile',\n 745: 'projector',\n 746: 'puck, hockey puck',\n 747: 'punching bag, punch bag, punching ball, punchball',\n 748: 'purse',\n 749: 'quill, quill pen',\n 750: 'quilt, comforter, comfort, puff',\n 751: 'racer, race car, racing car',\n 752: 'racket, racquet',\n 753: 'radiator',\n 754: 'radio, wireless',\n 755: 'radio telescope, radio reflector',\n 756: 'rain barrel',\n 757: 'recreational vehicle, RV, R.V.',\n 758: 'reel',\n 759: 'reflex camera',\n 760: 'refrigerator, icebox',\n 761: 'remote control, remote',\n 762: 'restaurant, eating house, eating place, eatery',\n 763: 'revolver, six-gun, six-shooter',\n 764: 'rifle',\n 765: 'rocking chair, rocker',\n 766: 'rotisserie',\n 767: 'rubber eraser, rubber, pencil eraser',\n 768: 'rugby ball',\n 769: 'rule, ruler',\n 770: 'running shoe',\n 771: 'safe',\n 772: 'safety pin',\n 773: 'saltshaker, salt shaker',\n 774: 'sandal',\n 775: 'sarong',\n 776: 'sax, saxophone',\n 777: 'scabbard',\n 778: 'scale, weighing machine',\n 779: 'school bus',\n 780: 'schooner',\n 781: 'scoreboard',\n 782: 'screen, CRT screen',\n 783: 'screw',\n 784: 'screwdriver',\n 785: 'seat belt, seatbelt',\n 786: 'sewing machine',\n 787: 'shield, buckler',\n 788: 'shoe shop, shoe-shop, shoe store',\n 789: 'shoji',\n 790: 'shopping basket',\n 791: 'shopping cart',\n 792: 'shovel',\n 793: 'shower cap',\n 794: 'shower curtain',\n 795: 'ski',\n 796: 'ski mask',\n 797: 'sleeping bag',\n 798: 'slide rule, slipstick',\n 799: 'sliding door',\n 800: 'slot, one-armed bandit',\n 801: 'snorkel',\n 802: 'snowmobile',\n 803: 'snowplow, snowplough',\n 804: 'soap dispenser',\n 805: 'soccer ball',\n 806: 'sock',\n 807: 'solar dish, solar collector, solar furnace',\n 808: 'sombrero',\n 809: 'soup bowl',\n 810: 'space bar',\n 811: 'space heater',\n 812: 'space shuttle',\n 813: 'spatula',\n 814: 'speedboat',\n 815: \"spider web, spider's web\",\n 816: 'spindle',\n 817: 'sports car, sport car',\n 818: 'spotlight, spot',\n 819: 'stage',\n 820: 'steam locomotive',\n 821: 'steel arch bridge',\n 822: 'steel drum',\n 823: 'stethoscope',\n 824: 'stole',\n 825: 'stone wall',\n 826: 'stopwatch, stop watch',\n 827: 'stove',\n 828: 'strainer',\n 829: 'streetcar, tram, tramcar, trolley, trolley car',\n 830: 'stretcher',\n 831: 'studio couch, day bed',\n 832: 'stupa, tope',\n 833: 'submarine, pigboat, sub, U-boat',\n 834: 'suit, suit of clothes',\n 835: 'sundial',\n 836: 'sunglass',\n 837: 'sunglasses, dark glasses, shades',\n 838: 'sunscreen, sunblock, sun blocker',\n 839: 'suspension bridge',\n 840: 'swab, swob, mop',\n 841: 'sweatshirt',\n 842: 'swimming trunks, bathing trunks',\n 843: 'swing',\n 844: 'switch, electric switch, electrical switch',\n 845: 'syringe',\n 846: 'table lamp',\n 847: 'tank, army tank, armored combat vehicle, armoured combat vehicle',\n 848: 'tape player',\n 849: 'teapot',\n 850: 'teddy, teddy bear',\n 851: 'television, television system',\n 852: 'tennis ball',\n 853: 'thatch, thatched roof',\n 854: 'theater curtain, theatre curtain',\n 855: 'thimble',\n 856: 'thresher, thrasher, threshing machine',\n 857: 'throne',\n 858: 'tile roof',\n 859: 'toaster',\n 860: 'tobacco shop, tobacconist shop, tobacconist',\n 861: 'toilet seat',\n 862: 'torch',\n 863: 'totem pole',\n 864: 'tow truck, tow car, wrecker',\n 865: 'toyshop',\n 866: 'tractor',\n 867: 'trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi',\n 868: 'tray',\n 869: 'trench coat',\n 870: 'tricycle, trike, velocipede',\n 871: 'trimaran',\n 872: 'tripod',\n 873: 'triumphal arch',\n 874: 'trolleybus, trolley coach, trackless trolley',\n 875: 'trombone',\n 876: 'tub, vat',\n 877: 'turnstile',\n 878: 'typewriter keyboard',\n 879: 'umbrella',\n 880: 'unicycle, monocycle',\n 881: 'upright, upright piano',\n 882: 'vacuum, vacuum cleaner',\n 883: 'vase',\n 884: 'vault',\n 885: 'velvet',\n 886: 'vending machine',\n 887: 'vestment',\n 888: 'viaduct',\n 889: 'violin, fiddle',\n 890: 'volleyball',\n 891: 'waffle iron',\n 892: 'wall clock',\n 893: 'wallet, billfold, notecase, pocketbook',\n 894: 'wardrobe, closet, press',\n 895: 'warplane, military plane',\n 896: 'washbasin, handbasin, washbowl, lavabo, wash-hand basin',\n 897: 'washer, automatic washer, washing machine',\n 898: 'water bottle',\n 899: 'water jug',\n 900: 'water tower',\n 901: 'whiskey jug',\n 902: 'whistle',\n 903: 'wig',\n 904: 'window screen',\n 905: 'window shade',\n 906: 'Windsor tie',\n 907: 'wine bottle',\n 908: 'wing',\n 909: 'wok',\n 910: 'wooden spoon',\n 911: 'wool, woolen, woollen',\n 912: 'worm fence, snake fence, snake-rail fence, Virginia fence',\n 913: 'wreck',\n 914: 'yawl',\n 915: 'yurt',\n 916: 'web site, website, internet site, site',\n 917: 'comic book',\n 918: 'crossword puzzle, crossword',\n 919: 'street sign',\n 920: 'traffic light, traffic signal, stoplight',\n 921: 'book jacket, dust cover, dust jacket, dust wrapper',\n 922: 'menu',\n 923: 'plate',\n 924: 'guacamole',\n 925: 'consomme',\n 926: 'hot pot, hotpot',\n 927: 'trifle',\n 928: 'ice cream, icecream',\n 929: 'ice lolly, lolly, lollipop, popsicle',\n 930: 'French loaf',\n 931: 'bagel, beigel',\n 932: 'pretzel',\n 933: 'cheeseburger',\n 934: 'hotdog, hot dog, red hot',\n 935: 'mashed potato',\n 936: 'head cabbage',\n 937: 'broccoli',\n 938: 'cauliflower',\n 939: 'zucchini, courgette',\n 940: 'spaghetti squash',\n 941: 'acorn squash',\n 942: 'butternut squash',\n 943: 'cucumber, cuke',\n 944: 'artichoke, globe artichoke',\n 945: 'bell pepper',\n 946: 'cardoon',\n 947: 'mushroom',\n 948: 'Granny Smith',\n 949: 'strawberry',\n 950: 'orange',\n 951: 'lemon',\n 952: 'fig',\n 953: 'pineapple, ananas',\n 954: 'banana',\n 955: 'jackfruit, jak, jack',\n 956: 'custard apple',\n 957: 'pomegranate',\n 958: 'hay',\n 959: 'carbonara',\n 960: 'chocolate sauce, chocolate syrup',\n 961: 'dough',\n 962: 'meat loaf, meatloaf',\n 963: 'pizza, pizza pie',\n 964: 'potpie',\n 965: 'burrito',\n 966: 'red wine',\n 967: 'espresso',\n 968: 'cup',\n 969: 'eggnog',\n 970: 'alp',\n 971: 'bubble',\n 972: 'cliff, drop, drop-off',\n 973: 'coral reef',\n 974: 'geyser',\n 975: 'lakeside, lakeshore',\n 976: 'promontory, headland, head, foreland',\n 977: 'sandbar, sand bar',\n 978: 'seashore, coast, seacoast, sea-coast',\n 979: 'valley, vale',\n 980: 'volcano',\n 981: 'ballplayer, baseball player',\n 982: 'groom, bridegroom',\n 983: 'scuba diver',\n 984: 'rapeseed',\n 985: 'daisy',\n 986: \"yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum\",\n 987: 'corn',\n 988: 'acorn',\n 989: 'hip, rose hip, rosehip',\n 990: 'buckeye, horse chestnut, conker',\n 991: 'coral fungus',\n 992: 'agaric',\n 993: 'gyromitra',\n 994: 'stinkhorn, carrion fungus',\n 995: 'earthstar',\n 996: 'hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa',\n 997: 'bolete',\n 998: 'ear, spike, capitulum',\n 999: 'toilet tissue, toilet paper, bathroom tissue'}\n" }, { "path": "examples/ONNX/resnet50/int8.py", "content": "import calibrator\nimport tensorrt as trt\n\n# Use TensorRT ONNX parser to parse model file, and enable INT8 calibration during engine construction\ndef build_int8_engine_onnx(model_file, image_dir, batch_size, calibration_batches, engine_file, cache_file='INT8CalibrationTable'):\n\n TRT_LOGGER = trt.Logger(trt.Logger.WARNING)\n with trt.Builder(TRT_LOGGER) as builder, builder.create_network() as network, trt.OnnxParser(network, TRT_LOGGER) as parser:\n # Load the Onnx model and parse it in order to populate the TensorRT network.\n with open(model_file, 'rb') as model:\n parser.parse(model.read())\n\n # Allow builder to use INT8 or FP16 kernels when building engine\n builder.int8_mode = True\n builder.fp16_mode = True\n calib = calibrator.ONNXEntropyCalibrator(image_dir, batch_size, calibration_batches, cache_file)\n builder.int8_calibrator = calib\n builder.max_batch_size = batch_size\n\n engine = builder.build_cuda_engine(network)\n\n with open(engine_file, 'wb') as f:\n f.write(engine.serialize())\n\n" }, { "path": "examples/ONNX/resnet50/onnx_utils.py", "content": "#!/usr/bin/env python3\nimport glob\nimport os\n\nimport onnx\nfrom onnx import numpy_helper\nfrom matplotlib import pyplot as plt\nimport numpy as np\n\ndef load_inputs(test_data_dir):\n # Load inputs\n inputs = []\n inputs_num = len(glob.glob(os.path.join(test_data_dir, 'input_*.pb')))\n for i in range(inputs_num):\n input_file = os.path.join(test_data_dir, 'input_{}.pb'.format(i))\n tensor = onnx.TensorProto()\n with open(input_file, 'rb') as f:\n tensor.ParseFromString(f.read())\n inputs.append(numpy_helper.to_array(tensor))\n return inputs\n\ndef load_outputs(test_data_dir):\n # Load reference outputs\n ref_outputs = []\n ref_outputs_num = len(glob.glob(os.path.join(test_data_dir, 'output_*.pb')))\n for i in range(ref_outputs_num):\n output_file = os.path.join(test_data_dir, 'output_{}.pb'.format(i))\n tensor = onnx.TensorProto()\n with open(output_file, 'rb') as f:\n tensor.ParseFromString(f.read())\n ref_outputs.append(numpy_helper.to_array(tensor))\n return ref_outputs\n\ndef mnist_image(data):\n two_d = (np.reshape(data, (28, 28))).astype(np.uint8)\n plt.imshow(two_d, interpolation='nearest')\n return plt\n\ndef softmax(x):\n \"\"\"Compute softmax values for each sets of scores in x.\"\"\"\n e_x = np.exp(x - np.max(x))\n return e_x / e_x.sum()\n" }, { "path": "examples/ONNX/resnet50/open_source_images.md5", "content": "6cd502bc217f3960cf34447ec4ede610 open_source_images.tar.gz\n" }, { "path": "examples/ONNX/resnet50/resnet50.md5", "content": "0e8088c7b1a1a9b2d0a5ae05601cc55e resnet50.tar.gz\n" }, { "path": "examples/ONNX/resnet50/run_jpeg_test.py", "content": "#!/usr/bin/env python3\n\nimport os\nimport time\n\nimport trtlab\nimport onnx_utils as utils\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport mxnet as mx\nfrom mxnet.gluon.data.vision import transforms\n\nfrom imagenet_labels import labels\n\nimport click\n\ntests = {}\n\ndef tensorrt_init(engines):\n manager = trtlab.InferenceManager(max_exec_concurrency=4)\n runners = []\n for engine in engines:\n name, _ = os.path.splitext(os.path.basename(engine))\n runners.append(manager.register_tensorrt_engine(name, engine))\n manager.update_resources()\n return runners\n\ndef infer_image(runner, image):\n inputs = preprocess_image(runner, image)\n future = runner.infer(**inputs)\n result = future.get()\n for name, tensor in result.items():\n tensor = tensor.reshape(1000)\n idx = np.argmax(tensor) \n print(\"\\n*** Results ***\") \n print(labels[idx], tensor[idx])\n print(\"\") \n\ndef preprocess_image(runner, image_path):\n inputs = runner.input_bindings()\n keys = list(inputs.keys())\n input_name = keys[0]\n img = np.array(plt.imread(image_path))\n img = transform_image(img)\n return { input_name: img }\n\ndef transform_image(img):\n transform_fn = transforms.Compose([\n transforms.Resize(224),\n transforms.CenterCrop(224),\n transforms.ToTensor(),\n transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n ])\n img = transform_fn(mx.nd.array(img)).asnumpy()\n img = np.expand_dims(img, axis=0) # batchify\n return img\n\n\ndef validate_results(computed, expected):\n keys = list(computed.keys())\n output_name = keys[0]\n output_value = computed[output_name]\n np.testing.assert_almost_equal(output_value, expected[0], decimal=3)\n print(\"-- Test Passed: All outputs {} match within 3 decimals\".format(output_value.shape))\n\nFile = click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True)\nPath = click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)\n\n@click.command()\n@click.option(\"--image\", type=File, multiple=True)\n@click.argument(\"engine\", type=File, nargs=1)\ndef main(engine, image):\n runners = tensorrt_init([engine])\n for runner in runners:\n for img in image:\n infer_image(runner, img) \n \n\nif __name__ == \"__main__\":\n main() \n" }, { "path": "examples/ONNX/resnet50/run_onnx_tests.py", "content": "#!/usr/bin/env python3\n\nimport os\n\nimport trtlab\nimport numpy as np\n\nimport click\nimport onnx_utils as utils\n\ntests = {}\n\ndef tensorrt_init(engines):\n manager = trtlab.InferenceManager(max_exec_concurrency=4)\n runners = []\n for engine in engines:\n name, _ = os.path.splitext(os.path.basename(engine))\n runners.append(manager.register_tensorrt_engine(name, engine))\n manager.update_resources()\n return runners\n\ndef test_data(test_path):\n for path, dirs, files in os.walk(test_path):\n if os.path.basename(path).startswith(\"test_\"):\n tests[path] = files \n for path, files in tests.items():\n inputs = utils.load_inputs(path)\n outputs = utils.load_outputs(path)\n print(\"** Testing {} **\".format(path))\n yield inputs, outputs\n\ndef run_test(runner, inputs, outputs):\n inputs = preprocess_inputs(runner, inputs)\n future = runner.infer(**inputs)\n result = future.get()\n validate_results(result, outputs)\n\ndef preprocess_inputs(runner, inputs):\n expected_input = runner.input_bindings()\n if len(expected_input) != len(inputs):\n raise RuntimeError(\"mismatched number of inputs\")\n keys = list(expected_input.keys())\n input_name = keys[0]\n info = expected_input[keys[0]]\n shape = info['shape']\n tensor = inputs[0]\n batch_size = tensor.shape[0]\n if list(shape) != list(tensor.shape[1:]):\n raise RuntimeError(\"mismatched input dimensions\")\n return { input_name: tensor }\n\ndef validate_results(computed, expected):\n keys = list(computed.keys())\n output_name = keys[0]\n output_value = computed[output_name]\n np.testing.assert_almost_equal(output_value, expected[0], decimal=3)\n print(\"-- Test Passed: All outputs {} match within 3 decimals\".format(output_value.shape))\n\nFile = click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True)\nPath = click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True)\n\n@click.command()\n@click.option(\"--tests\", type=Path, default=\"resnet50\")\n@click.argument(\"engine\", type=File, nargs=1)\ndef main(engine, tests):\n runners = tensorrt_init([engine])\n for runner in runners:\n for inputs, outputs in test_data(tests):\n run_test(runner, inputs, outputs)\n \n\nif __name__ == \"__main__\":\n main() \n" }, { "path": "examples/nvRPC/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nadd_subdirectory(UnaryService)\nadd_subdirectory(StreamingService)\nadd_subdirectory(SharedMemoryService)\n\n# TODO: WIP\n# add_subdirectory(StreamingInOrderSendRecv)" }, { "path": "examples/nvRPC/SharedMemoryService/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nadd_executable(sysv-nvrpc.x\n server.cc)\n\ntarget_link_libraries(sysv-nvrpc.x\n nvrpc\n echo-protos\n gflags\n)\n\nadd_executable(sysv-client.x\n client.cc)\n\ntarget_link_libraries(sysv-client.x\n nvrpc\n echo-protos\n gflags\n)\n" }, { "path": "examples/nvRPC/SharedMemoryService/README.md", "content": "# Shared Memory Service\n\nClient/Server service extending the Basic nvRPC example.\n\nThe client (`sysv-client.x`) creates a `CyclicAllocator` from which it\nallocates buffers of shared memory.\n\nThe client:\n 1. Write some data into shared memory (batch_id and 0xDEADBEEF), `data[0]` and `data[1]` respectively.\n 2. Packs the RPC message with the batch_id and the SystemV memory descriptor details\n 3. Initiates the RPC\n 4. Checks the value of the address in which it wrote 0xDEADBEEF to ensure that the server wrote it's response to the proper location.\n\nOn reciept of a client RPC, the server Aquires a `Descriptor` to the\noffset in the shared memory segment required in the RPC. This is done via the\n`ExternalSharedMemoryManager`.\n\nThe server:\n 1. Aquires the `Descriptor` to the offset\n 2. Checks the values at `data[0]` and `data[1]` for `batch_id` and `0xDEADBEEF` respectively\n 3. Write the `batch_id` into `[1]` element\n 4. Returns the Response" }, { "path": "examples/nvRPC/SharedMemoryService/client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"tensorrt/laboratory/core/memory/cyclic_allocator.h\"\n#include \"tensorrt/laboratory/core/memory/system_v.h\"\n\n#include \"echo.grpc.pb.h\"\n\nusing grpc::Channel;\nusing grpc::ClientContext;\nusing grpc::Status;\nusing simple::Inference;\nusing simple::Input;\nusing simple::Output;\n\nusing trtlab::CyclicAllocator;\nusing trtlab::SystemV;\n\nstatic constexpr size_t one_mb = 1024 * 1024;\n\nDEFINE_int32(count, 1, \"number of grpc messages to send\");\n\nclass SimpleClient final\n{\n public:\n SimpleClient(std::shared_ptr channel)\n : m_Stub(Inference::NewStub(channel)), m_Memory(5, one_mb)\n {\n }\n\n // Generate and send RPC message\n int Compute(const int batch_id)\n {\n // Allocate some SysV shared memory from the CyclicAllocator\n CyclicAllocator::Descriptor mdesc = RandomAllocation();\n\n // Populate the request object\n Input request;\n request.set_batch_id(batch_id);\n auto sysv = request.mutable_sysv();\n sysv->set_shm_id(mdesc->Stack().Memory().ShmID());\n sysv->set_offset(mdesc->Offset());\n sysv->set_size(mdesc->Size());\n\n // Write the batch_id to the shared memory segment\n // This will validated against the batch_id in the message body on the server\n auto data = mdesc->CastToArray();\n data[0] = batch_id;\n data[1] = 0xDEADBEEF;\n\n // Container for the data we expect from the server.\n Output reply;\n\n // Context for the client. It could be used to convey extra information to\n // the server and/or tweak certain RPC behaviors.\n ClientContext context;\n\n // The actual RPC.\n Status status = m_Stub->Compute(&context, request, &reply);\n\n if(status.ok())\n {\n CHECK_EQ(data[1], batch_id);\n return reply.batch_id();\n }\n else\n {\n LOG(ERROR) << status.error_code() << \": \" << status.error_message();\n return -1;\n }\n }\n\n private:\n CyclicAllocator::Descriptor RandomAllocation()\n {\n size_t bytes = rand() % (m_Memory.MaxAllocationSize() / 4);\n bytes = std::max(bytes, 16UL); // guarantee at least 16 bytes (2x size_t)\n DLOG(INFO) << \"RandomAllocation: \" << bytes << \" bytes\";\n return m_Memory.Allocate(bytes);\n }\n\n std::unique_ptr m_Stub;\n CyclicAllocator m_Memory;\n};\n\nint main(int argc, char** argv)\n{\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n SimpleClient client(grpc::CreateChannel(\"localhost:50051\", grpc::InsecureChannelCredentials()));\n\n auto start = std::chrono::steady_clock::now();\n for(int i = 0; i < FLAGS_count; i++)\n {\n auto reply = client.Compute(i);\n LOG_IF(INFO, reply == -1) << \"BatchId received: \" << reply;\n }\n auto end = std::chrono::steady_clock::now();\n float elapsed = std::chrono::duration(end - start).count();\n std::cout << FLAGS_count << \" requests in \" << elapsed << \"seconds\" << std::endl;\n return 0;\n}\n" }, { "path": "examples/nvRPC/SharedMemoryService/server.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \n#include \n\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n#include \"tensorrt/laboratory/core/memory/descriptor.h\"\n#include \"tensorrt/laboratory/core/memory/system_v.h\"\n#include \"tensorrt/laboratory/core/pool.h\"\n#include \"tensorrt/laboratory/core/resources.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n#include \"tensorrt/laboratory/core/utils.h\"\n\n#include \"echo.grpc.pb.h\"\n#include \"echo.pb.h\"\n\nusing nvrpc::AsyncRPC;\nusing nvrpc::AsyncService;\nusing nvrpc::Context;\nusing nvrpc::Executor;\nusing nvrpc::Server;\nusing trtlab::Resources;\nusing trtlab::ThreadPool;\n\nusing trtlab::Descriptor;\nusing trtlab::SystemV;\n\n// CLI Options\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\n\n/**\n * @brief SystemV Memory Manager\n *\n * This object does not allocate system v shared memory segments. Instead, it attaches and manages\n * descriptors into shared memory segments allocated by an external source.\n */\nclass ExternalSharedMemoryManager final\n{\n class PartialSegmentDescriptor final : public Descriptor\n {\n public:\n PartialSegmentDescriptor(const std::shared_ptr& segment, size_t offset,\n size_t size)\n : Descriptor((*segment)[offset], size, \"PartialSysVSegment\"),\n m_Segment(segment)\n {\n }\n\n PartialSegmentDescriptor(PartialSegmentDescriptor&& other)\n : Descriptor(std::move(other)), m_Segment{\n std::exchange(other.m_Segment, nullptr)}\n {\n }\n\n PartialSegmentDescriptor& operator=(PartialSegmentDescriptor&&) = delete;\n DELETE_COPYABILITY(PartialSegmentDescriptor);\n\n virtual ~PartialSegmentDescriptor() override {}\n\n private:\n std::shared_ptr m_Segment;\n };\n\n public:\n ExternalSharedMemoryManager() = default;\n using Descriptor = std::unique_ptr;\n\n Descriptor Acquire(size_t shm_id, size_t offset, size_t size)\n {\n const auto& segment = GetOrAttachToShmID(shm_id);\n CHECK_LE(offset + size, segment->Size());\n return std::make_unique(segment, offset, size);\n }\n\n void Release(size_t shm_id)\n {\n std::lock_guard l(m_Mutex);\n auto count = m_AttachedSegments.erase(shm_id);\n DLOG_IF(WARNING, count == 0) << \"Attempting to Release an unmapped shm_id\";\n }\n\n protected:\n std::shared_ptr GetOrAttachToShmID(size_t shm_id)\n {\n std::shared_ptr segment;\n std::lock_guard l(m_Mutex);\n auto search = m_AttachedSegments.find(shm_id);\n if(search == m_AttachedSegments.end())\n {\n DLOG(INFO) << \"SystemV Manager: attaching to shm_id: \" << shm_id;\n segment = SystemV::Attach(shm_id);\n m_AttachedSegments[shm_id] = segment;\n }\n else\n {\n segment = search->second;\n }\n return segment;\n }\n\n private:\n std::map> m_AttachedSegments;\n std::mutex m_Mutex;\n};\n\nstruct SimpleResources : public Resources\n{\n SimpleResources() = default;\n\n ExternalSharedMemoryManager& GetExternalSharedMemoryManager()\n {\n return m_ExternalSharedMemoryManager;\n }\n\n private:\n ExternalSharedMemoryManager m_ExternalSharedMemoryManager;\n};\n\nclass SimpleContext final : public Context\n{\n void ExecuteRPC(RequestType& input, ResponseType& output) final override\n {\n ExternalSharedMemoryManager::Descriptor mdesc;\n if(input.has_sysv())\n {\n mdesc = GetResources()->GetExternalSharedMemoryManager().Acquire(\n input.sysv().shm_id(), input.sysv().offset(), input.sysv().size());\n }\n CHECK(mdesc);\n auto array = mdesc->CastToArray();\n CHECK_EQ(array[0], input.batch_id());\n CHECK_EQ(array[1], 0xDEADBEEF);\n array[1] = input.batch_id();\n\n output.set_batch_id(input.batch_id());\n this->FinishResponse();\n }\n};\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n\n ::google::InitGoogleLogging(\"simpleServer\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n // A server will bind an IP:PORT to listen on\n Server server(\"0.0.0.0:50051\");\n\n // A server can host multiple services\n LOG(INFO) << \"Register Service (simple::Inference) with Server\";\n auto simpleInference = server.RegisterAsyncService();\n\n LOG(INFO) << \"Register RPC (simple::Inference::Compute) with Service (simple::Inference)\";\n auto rpcCompute = simpleInference->RegisterRPC(\n &simple::Inference::AsyncService::RequestCompute);\n\n LOG(INFO) << \"Initializing Resources for RPC (simple::Inference::Compute)\";\n auto rpcResources = std::make_shared();\n\n LOG(INFO) << \"Creating Executor\";\n auto executor = server.RegisterExecutor(new Executor(1));\n\n LOG(INFO) << \"Creating Execution Contexts for RPC (simple::Inference::Compute) with Executor\";\n executor->RegisterContexts(rpcCompute, rpcResources, 10);\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(2000), [] {});\n}\n" }, { "path": "examples/nvRPC/StreamingInOrderSendRecv/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nadd_executable(nvrpc-bidirectional-server.x\n server.cc)\n\ntarget_link_libraries(nvrpc-bidirectional-server.x\n nvrpc\n echo-protos\n gflags\n)\n\nadd_executable(nvrpc-bidirectional-client.x\n client.cc\n)\n\ntarget_link_libraries(nvrpc-bidirectional-client.x\n nvrpc\n nvrpc-client\n echo-protos\n gflags\n)" }, { "path": "examples/nvRPC/StreamingInOrderSendRecv/README.md", "content": "# BidirectionalStream In-Order Send/Recv\n\n```\nrpc InOrderSendRecv (stream Request) returns (stream Response)\n```\n\nThe service will accept a stream or Requests, queue them for in-order execution\nvia the `ExecuteRPC` virtual method, and for each result of `ExecuteRPC` return\na Response on the stream.\n\nOnly one `ExecuteRPC` can be in-action at anytime which allows the RPC to\noptionally maintain a state. In some regards, this lifecyle could be used to\nmodel an RNN as the collective history can be built into the resources.\n\n" }, { "path": "examples/nvRPC/StreamingInOrderSendRecv/client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"nvrpc/client/client_streaming.h\"\n#include \"nvrpc/client/executor.h\"\n\nusing grpc::Channel;\nusing grpc::ClientContext;\nusing grpc::Status;\n\nusing nvrpc::client::ClientBidirectional;\nusing nvrpc::client::Executor;\n\n#include \"echo.grpc.pb.h\"\n\nusing simple::Inference;\nusing simple::Input;\nusing simple::Output;\n\nDEFINE_int32(count, 100, \"number of grpc messages to send\");\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\nDEFINE_string(hostname, \"127.0.0.1:50051\", \"hostname and port\");\n\nint main(int argc, char** argv)\n{\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n std::mutex mutex;\n std::size_t count = 0;\n\n auto executor = std::make_shared(FLAGS_thread_count);\n\n auto channel = grpc::CreateChannel(FLAGS_hostname, grpc::InsecureChannelCredentials());\n auto stub = Inference::NewStub(channel);\n\n auto infer_prepare_fn = [&stub](::grpc::ClientContext * context,\n ::grpc::CompletionQueue * cq) -> auto\n {\n return std::move(stub->PrepareAsyncBidirectional(context, cq));\n };\n\n auto stream = std::make_unique>(\n infer_prepare_fn, executor,\n [](Input&& request) {\n LOG_FIRST_N(INFO, 10) << \"Sent Request with BatchID: \" << request.batch_id();\n static size_t last = 0;\n CHECK_EQ(last + 1, request.batch_id());\n ++last;\n // CHECK(request.batch_id());\n },\n [&mutex, &count](Output&& response) {\n LOG_FIRST_N(INFO, 10) << \"Received Response with BatchID: \" << response.batch_id();\n // CHECK(response.batch_id());\n std::lock_guard lock(mutex);\n --count;\n });\n\n auto start = std::chrono::steady_clock::now();\n auto elapsed = [start]() -> float {\n return std::chrono::duration(std::chrono::steady_clock::now() - start).count();\n };\n\n for(int i = 1; i < FLAGS_count + 1; i++)\n {\n {\n std::lock_guard lock(mutex);\n ++count;\n }\n Input input;\n input.set_batch_id(i);\n stream->Send(std::move(input));\n }\n std::cout << FLAGS_count << \" queued in \" << elapsed() << \"seconds\" << std::endl;\n auto future = stream->Done();\n auto status = future.get();\n executor->ShutdownAndJoin();\n CHECK_EQ(count, 0UL);\n std::cout << FLAGS_count << \" completed in \" << elapsed() << \"seconds\" << std::endl;\n std::cout << \"gRPC Status: \" << (status.ok() ? \"OK\" : \"NOT OK\") << std::endl;\n return 0;\n}" }, { "path": "examples/nvRPC/StreamingInOrderSendRecv/server.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \"tensorrt/laboratory/core/pool.h\"\n#include \"tensorrt/laboratory/core/resources.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n\nusing trtlab::Resources;\nusing trtlab::ThreadPool;\n\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n\nusing nvrpc::AsyncRPC;\nusing nvrpc::AsyncService;\nusing nvrpc::BidirectionalContext;\nusing nvrpc::Executor;\nusing nvrpc::Server;\n\n#include \"echo.grpc.pb.h\"\n#include \"echo.pb.h\"\n\n// CLI Options\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\n\n// Define the resources your RPC will need to execute\n// ==================================================\n// In this case, all simple::Inference::Compute RPCs share a threadpool in which they will\n// queue up some work on. This essentially means, after the message as been received and\n// processed, the actual work for the RPC is pushed to a worker pool outside the scope of\n// the transaction processing system (TPS). This is essentially async computing, we have\n// decoupled the transaction from the workers executing the implementation. The TPS can\n// continue to queue work, while the workers process the load.\nstruct SimpleResources : public Resources\n{\n SimpleResources(int numThreadsInPool = 3) : m_ThreadPool(numThreadsInPool)\n {\n LOG(INFO) << \"Server ThreadCount: \" << numThreadsInPool;\n }\n\n ThreadPool& AcquireThreadPool() { return m_ThreadPool; }\n\n private:\n ThreadPool m_ThreadPool;\n};\n\n// Contexts hold the state and provide the definition of the work to be performed by the RPC.\n// This is where you define what gets executed for a given RPC.\n// Incoming Message = simple::Input (RequestType)\n// Outgoing Message = simple::Output (ResponseType)\nclass SimpleContext final\n : public BidirectionalContext\n{\n void ExecuteRPC(RequestType& input, ResponseType& output) final override\n {\n // We could do work here, but we'd block the TPS, i.e. the threads pulling messages\n // off the incoming recieve queue. Very quick responses are best done here; however,\n // longer running workload should be offloaded so the TPS can avoid being blocked.\n // GetResources()->AcquireThreadPool().enqueue([this, &input, &output]{\n // Now running on a worker thread of the ThreadPool defined in SimpleResources.\n // Here we are just echoing back the incoming // batch_id; however, in later\n // examples, we'll show how to run an async cuda pipline.\n LOG_FIRST_N(INFO, 10) << \"BatchID: \" << input.batch_id() << \" Tag = \" << Tag()\n << \" Thread = \" << std::this_thread::get_id();\n output.set_batch_id(input.batch_id());\n this->FinishResponse();\n // });\n // The TPS thread is now free to continue processing message - async ftw!\n }\n};\n\nDEFINE_string(ip_port, \"0.0.0.0:50051\", \"IP/Port\");\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n\n ::google::InitGoogleLogging(\"simpleServer\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n // A server will bind an IP:PORT to listen on\n Server server(FLAGS_ip_port);\n\n // A server can host multiple services\n LOG(INFO) << \"Register Service (simple::Inference) with Server\";\n auto simpleInference = server.RegisterAsyncService();\n\n // An RPC has two components that need to be specified when registering with the service:\n // 1) Type of Execution Context (SimpleContext). The execution context defines the behavor\n // of the RPC, i.e. it contains the control logic for the execution of the RPC.\n // 2) The Request function (RequestCompute) which was generated by gRPC when compiling the\n // protobuf which defined the service. This function is responsible for queuing the\n // RPC's execution context to the\n LOG(INFO) << \"Register RPC (simple::Inference::Compute) with Service (simple::Inference)\";\n auto rpcCompute = simpleInference->RegisterRPC(\n &simple::Inference::AsyncService::RequestBidirectional);\n\n LOG(INFO) << \"Initializing Resources for RPC (simple::Inference::Compute)\";\n auto rpcResources = std::make_shared(FLAGS_thread_count);\n\n // Create Executors - Executors provide the messaging processing resources for the RPCs\n // Multiple Executors can be registered with a Server. The executor is responsible\n // for pulling incoming message off the receive queue and executing the associated\n // context. By default, an executor only uses a single thread. A typical usecase is\n // an Executor executes a context, which immediate pushes the work to a thread pool.\n // However, for very low-latency messaging, you might want to use a multi-threaded\n // Executor and a Blocking Context - meaning the Context performs the entire RPC function\n // on the Executor's thread.\n LOG(INFO) << \"Creating Executor\";\n auto executor = server.RegisterExecutor(new Executor(1));\n\n // You can register RPC execution contexts from any registered RPC on any executor.\n // The power of that will become clear in later examples. For now, we will register\n // 10 instances of the simple::Inference::Compute RPC's SimpleContext execution context\n // with the Executor.\n LOG(INFO) << \"Creating Execution Contexts for RPC (simple::Inference::Compute) with Executor\";\n executor->RegisterContexts(rpcCompute, rpcResources, 10);\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(2000), [] {\n // This is a timeout loop executed every 2seconds\n // Run() with no arguments will run an empty timeout loop every 5 seconds.\n // RunAsync() will return immediately, its your responsibility to ensure the\n // server doesn't go out of scope or a Shutdown will be triggered on your services.\n });\n}\n" }, { "path": "examples/nvRPC/StreamingInOrderSendRecv/test.sh", "content": "#!/bin/bash \n\ncleanup() {\n kill $(jobs -p) ||:\n}\ntrap \"cleanup\" EXIT SIGINT SIGTERM\n\n./nvrpc-bidirectional-server.x --ip_port=\"0.0.0.0:5555\" &\n\nf=$(mktmp)\ncat < $f\nPS1='nvRPC Bidirectional: '\n\ngo() { ./nvrpc-bidirectional-client.x --hostname=\"localhost:5555\" --count=${1:-100} }\nEOF\n\nps aux\n\necho\necho 'Try ./nvrpc-bidirectional-client.x --hostname=\"localhost:5555\" --count=100'\nbash --rcfile <(echo \"PS1='nvRPC Bidirectional: '\")\n" }, { "path": "examples/nvRPC/StreamingService/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nset(LIBS nvrpc echo-protos gflags)\n\nadd_executable(nvrpc-ping-pong-server.x ping-pong.cc)\nadd_executable(nvrpc-even-odds-server.x even-odds.cc)\n\ntarget_link_libraries(nvrpc-ping-pong-server.x ${LIBS})\ntarget_link_libraries(nvrpc-even-odds-server.x ${LIBS})\n\nadd_executable(nvrpc-streaming-client.x client.cc)\ntarget_link_libraries(nvrpc-streaming-client.x ${LIBS} nvrpc-client)" }, { "path": "examples/nvRPC/StreamingService/README.md", "content": "# Streaming Examples\n\nAsync gRPC streaming can take on many forms. nvRPC provides a set of LifeCycles\nto accommodate a variety of common use-cases.\n\nFor all examples, the RPC that we will implement will have the same form:\n\n```protobuf\nrpc Bidirectional (stream Input) returns (stream Output) {}\n```\n\nTo implement a `StreamingContext`, you must implement the `ReceivedRequest` pure\nvirtual method. This method is triggered once for each incoming request that is\nread from the stream. The `ServerStream` object is used to write responses or \nclose the stream. \n\n```c++\nclass SimpleContext final : public StreamingContext\n{\n void RequestReceived(RequestType&& input, std::shared_ptr stream) final override\n {\n // `input` incoming message of Input/RequestType\n // `stream` allows you to:\n // ->StreamID()\n // - unique identifier for this stream; note, the ID will be reused when the context is\n // recycled.\n // ->WriteResponse(Output&&)\n // - writes a response on the stream; returns `true` if the stream is connected;\n // otherwise, `false` if the stream is disconnected\n // ->IsConnected()\n // - bool - is the stream still connected to the client. `FinishStream` or `CancelStream`\n // will disconnect all `ServerStream` objects that share the same StreamID.\n // ->FinishStream()\n // - Close the Stream from the server-side with status OK\n // ->CancelStream()\n // - Close the Stream with status CANCELLED\n //\n // NOTE: The gRPC stream will stay open to the client until:\n // 1) the client closes its half of the stream, and\n // 2) all `ServerStream` objects are destroyed OR\n // the stream is explicitly closed by Cancel/FinishStream\n }\n```\n\nThe final comment is worth further discussion. The life of the `ServerStream` object does\nnot have to be tied to the life of the `ReceivedRequest` call. You can pass the\n`ServerStream` object off to an external resource which can then write messages to on the\nstream has long as the stream remains connected. \n\nThe stream will disconnect implicitly \n\n\n- `ping-pong.cc` - In-order send/recv stream. This client sends a Request and\n the server Response with the same value for `batch_id`. " }, { "path": "examples/nvRPC/StreamingService/client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"nvrpc/client/client_streaming.h\"\n#include \"nvrpc/client/executor.h\"\n\nusing grpc::Channel;\nusing grpc::ClientContext;\nusing grpc::Status;\n\nusing nvrpc::client::ClientStreaming;\nusing nvrpc::client::Executor;\n\n#include \"echo.grpc.pb.h\"\n\nusing simple::Inference;\nusing simple::Input;\nusing simple::Output;\n\nstatic bool ValidateEven(const char* flagname, int value)\n{\n LOG_IF(ERROR, value % 2) << \"Examples require an even number of messages\";\n return (value % 2 == 0);\n}\n\nDEFINE_int32(count, 100, \"number of grpc messages to send\");\nDEFINE_validator(count, &ValidateEven);\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\nDEFINE_string(hostname, \"127.0.0.1:50051\", \"hostname and port\");\n\nint main(int argc, char** argv)\n{\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n std::mutex mutex;\n std::size_t count = 0;\n\n auto executor = std::make_shared(FLAGS_thread_count);\n\n auto channel = grpc::CreateChannel(FLAGS_hostname, grpc::InsecureChannelCredentials());\n auto stub = Inference::NewStub(channel);\n\n auto infer_prepare_fn = [&stub](::grpc::ClientContext * context,\n ::grpc::CompletionQueue * cq) -> auto\n {\n return std::move(stub->PrepareAsyncBidirectional(context, cq));\n };\n\n auto stream = std::make_unique>(\n infer_prepare_fn, executor,\n [](Input&& request) {\n LOG_FIRST_N(INFO, 10) << \"Sent Request with BatchID: \" << request.batch_id();\n },\n [&mutex, &count](Output&& response) {\n static size_t last = 0;\n LOG_FIRST_N(INFO, 10) << \"Received Response with BatchID: \" << response.batch_id();\n CHECK_EQ(++last, response.batch_id());\n std::lock_guard lock(mutex);\n --count;\n });\n\n auto start = std::chrono::steady_clock::now();\n auto elapsed = [start]() -> float {\n return std::chrono::duration(std::chrono::steady_clock::now() - start).count();\n };\n\n for(int i = 1; i < FLAGS_count + 1; i++)\n {\n {\n std::lock_guard lock(mutex);\n ++count;\n }\n Input input;\n input.set_batch_id(i);\n stream->Write(std::move(input));\n }\n std::cout << FLAGS_count << \" queued in \" << elapsed() << \"seconds\" << std::endl;\n auto future = stream->Done();\n // auto future = stream->Status();\n auto status = future.get();\n std::cout << FLAGS_count << \" completed in \" << elapsed() << \"seconds\" << std::endl;\n std::cout << \"gRPC Status: \" << (status.ok() ? \"OK\" : \"NOT OK\") << std::endl;\n executor->ShutdownAndJoin();\n CHECK_EQ(count, 0UL);\n return 0;\n}" }, { "path": "examples/nvRPC/StreamingService/common.h", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#pragma once\n\n#include \n#include \n#include \n#include \n\n#include \"tensorrt/laboratory/core/pool.h\"\n#include \"tensorrt/laboratory/core/resources.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n\nusing trtlab::Resources;\nusing trtlab::ThreadPool;\n\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n\nusing nvrpc::AsyncRPC;\nusing nvrpc::AsyncService;\nusing nvrpc::Executor;\nusing nvrpc::Server;\nusing nvrpc::StreamingContext;\n\n#include \"echo.grpc.pb.h\"\n#include \"echo.pb.h\"\n" }, { "path": "examples/nvRPC/StreamingService/even-odds.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \"common.h\"\n\n// clang-format off\nstruct SimpleResources : public Resources\n{\n SimpleResources(int numThreadsInPool = 3) : m_ThreadPool(numThreadsInPool) {}\n ThreadPool& AcquireThreadPool() { return m_ThreadPool; }\n private:\n ThreadPool m_ThreadPool;\n};\n// clang-format on\n\nclass SimpleContext final : public StreamingContext\n{\n void RequestReceived(RequestType&& input, std::shared_ptr stream) final override\n {\n LOG_FIRST_N(INFO, 10) << \"BatchID: \" << input.batch_id() << \" Tag = \" << Tag()\n << \" Thread = \" << std::this_thread::get_id();\n\n // If even, send back two responses.\n // If odd, do nothing\n if(input.batch_id() % 2 == 0)\n {\n LOG_FIRST_N(INFO, 5) << \"Received Even an BatchID: Sending back two responses\";\n for(int i = input.batch_id() - 1; i <= input.batch_id(); i++)\n {\n ResponseType output;\n output.set_batch_id(i);\n stream->WriteResponse(std::move(output));\n }\n }\n else\n {\n LOG_FIRST_N(INFO, 5) << \"Received an Odd BatchID: No Response will be sent\";\n }\n }\n};\n\n// CLI Options\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\nDEFINE_string(ip_port, \"0.0.0.0:50051\", \"IP/Port\");\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"simpleServer\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n Server server(FLAGS_ip_port);\n auto simpleInference = server.RegisterAsyncService();\n auto rpcCompute = simpleInference->RegisterRPC(\n &simple::Inference::AsyncService::RequestBidirectional);\n auto rpcResources = std::make_shared(FLAGS_thread_count);\n auto executor = server.RegisterExecutor(new Executor(1));\n executor->RegisterContexts(rpcCompute, rpcResources, 10);\n server.Run(std::chrono::milliseconds(2000), [] {});\n}\n" }, { "path": "examples/nvRPC/StreamingService/ping-pong.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \"common.h\"\n\n// clang-format off\nstruct SimpleResources : public Resources\n{\n SimpleResources(int numThreadsInPool = 3) : m_ThreadPool(numThreadsInPool) {}\n ThreadPool& AcquireThreadPool() { return m_ThreadPool; }\n private:\n ThreadPool m_ThreadPool;\n};\n// clang-format on\n\nclass SimpleContext final : public StreamingContext\n{\n void RequestReceived(RequestType&& input, std::shared_ptr stream) final override\n {\n LOG_FIRST_N(INFO, 10) << \"BatchID: \" << input.batch_id() << \" Tag = \" << Tag()\n << \" Thread = \" << std::this_thread::get_id();\n ResponseType output;\n output.set_batch_id(input.batch_id());\n stream->WriteResponse(std::move(output));\n // TODO: add a test which the server closes the before the client does\n // stream->FinishStream();\n }\n};\n\n// CLI Options\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\nDEFINE_string(ip_port, \"0.0.0.0:50051\", \"IP/Port\");\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n ::google::InitGoogleLogging(\"simpleServer\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n Server server(FLAGS_ip_port);\n auto simpleInference = server.RegisterAsyncService();\n auto rpcCompute = simpleInference->RegisterRPC(\n &simple::Inference::AsyncService::RequestBidirectional);\n auto rpcResources = std::make_shared(FLAGS_thread_count);\n auto executor = server.RegisterExecutor(new Executor(1));\n executor->RegisterContexts(rpcCompute, rpcResources, 10);\n server.Run(std::chrono::milliseconds(2000), [] {});\n}\n" }, { "path": "examples/nvRPC/StreamingService/test.sh", "content": "#!/bin/bash \n\ncleanup() {\n kill $(jobs -p) ||:\n}\ntrap \"cleanup\" EXIT SIGINT SIGTERM\n\nexport PATH=\".:$PATH\"\n\nexe=${1:-\"./nvrpc-ping-pong-server.x\"}\n\n$exe --ip_port=\"0.0.0.0:5555\" &\n\nf=$(mktmp)\ncat < $f\nPS1='nvRPC Bidirectional: '\n\ngo() { ./nvrpc-streaming-client.x --hostname=\"localhost:5555\" --count=${1:-100} }\nEOF\n\nps aux\n\necho\necho 'Try ./nvrpc-streaming-client.x --hostname=\"localhost:5555\" --count=100'\nbash --rcfile <(echo \"PS1='nvRPC StreamingService: '\")\n" }, { "path": "examples/nvRPC/UnaryService/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nadd_executable(nvrpc-unary-server.x\n server.cc)\n\ntarget_link_libraries(nvrpc-unary-server.x\n nvrpc\n echo-protos\n gflags\n)\n\nadd_executable(nvrpc-unary-client.x\n client.cc)\n\ntarget_link_libraries(nvrpc-unary-client.x\n nvrpc\n nvrpc-client\n echo-protos\n gflags\n)" }, { "path": "examples/nvRPC/UnaryService/client.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \"nvrpc/client/client_unary.h\"\n#include \"nvrpc/client/executor.h\"\n\nusing grpc::Channel;\nusing grpc::ClientContext;\nusing grpc::Status;\n\nusing nvrpc::client::ClientUnary;\nusing nvrpc::client::Executor;\n\n#include \"echo.grpc.pb.h\"\n\nusing simple::Inference;\nusing simple::Input;\nusing simple::Output;\n\nDEFINE_int32(count, 100, \"number of grpc messages to send\");\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\n\nint main(int argc, char** argv)\n{\n // Instantiate the client. It requires a channel, out of which the actual RPCs\n // are created. This channel models a connection to an endpoint (in this case,\n // localhost at port 50051). We indicate that the channel isn't authenticated\n // (use of InsecureChannelCredentials()).\n FLAGS_alsologtostderr = 1; // It will dump to console\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n auto executor = std::make_shared(FLAGS_thread_count);\n\n auto channel = grpc::CreateChannel(\"localhost:50051\", grpc::InsecureChannelCredentials());\n auto stub = Inference::NewStub(channel);\n\n auto infer_prepare_fn = [&stub](::grpc::ClientContext * context, const ::simple::Input& request,\n ::grpc::CompletionQueue* cq) -> auto\n {\n return std::move(stub->PrepareAsyncCompute(context, request, cq));\n };\n\n auto runner = std::make_unique>(infer_prepare_fn, executor);\n\n auto start = std::chrono::steady_clock::now();\n auto elapsed = [start]() -> float {\n return std::chrono::duration(std::chrono::steady_clock::now() - start).count();\n };\n\n for(int i = 0; i < FLAGS_count; i++)\n {\n Input input;\n input.set_batch_id(i);\n runner->Enqueue(std::move(input),\n [i](Input& input, Output& output, ::grpc::Status& status) -> bool {\n CHECK(output.batch_id() == i);\n LOG_FIRST_N(INFO, 20) << \"Check: \" << i;\n return (bool)(output.batch_id() == i);\n });\n }\n std::cout << FLAGS_count << \" queued in \" << elapsed() << \"seconds\" << std::endl;\n executor->ShutdownAndJoin();\n std::cout << FLAGS_count << \" completed in \" << elapsed() << \"seconds\" << std::endl;\n return 0;\n}" }, { "path": "examples/nvRPC/UnaryService/server.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n#include \n#include \n#include \n\n#include \"tensorrt/laboratory/core/pool.h\"\n#include \"tensorrt/laboratory/core/resources.h\"\n#include \"tensorrt/laboratory/core/thread_pool.h\"\n\nusing trtlab::Resources;\nusing trtlab::ThreadPool;\n\n#include \"nvrpc/executor.h\"\n#include \"nvrpc/server.h\"\n#include \"nvrpc/service.h\"\n\nusing nvrpc::AsyncRPC;\nusing nvrpc::AsyncService;\nusing nvrpc::Context;\nusing nvrpc::Executor;\nusing nvrpc::Server;\n\n#include \"echo.grpc.pb.h\"\n#include \"echo.pb.h\"\n\n// CLI Options\nDEFINE_int32(thread_count, 1, \"Size of thread pool\");\n\n/**\n * Embedding a copy of the Protobuf specification for the gRPC service.\n *\n * Package Name: simple\n * Service Name: Inference\n * RPC Name: Compute\n *\n * Incoming Message: Input\n * Outgoing Message: Ouput\n **\n\nsyntax = \"proto3\";\n\npackage simple;\n\nservice Inference {\n rpc Compute (Input) returns (Output) {}\n}\n\nmessage Input {\n uint64 batch_id = 1;\n}\n\nmessage Output {\n uint64 batch_id = 1;\n}\n*/\n\n// Define the resources your RPC will need to execute\n// ==================================================\n// In this case, all simple::Inference::Compute RPCs share a threadpool in which they will\n// queue up some work on. This essentially means, after the message as been received and\n// processed, the actual work for the RPC is pushed to a worker pool outside the scope of\n// the transaction processing system (TPS). This is essentially async computing, we have\n// decoupled the transaction from the workers executing the implementation. The TPS can\n// continue to queue work, while the workers process the load.\nstruct SimpleResources : public Resources\n{\n SimpleResources(int numThreadsInPool = 3) : m_ThreadPool(numThreadsInPool)\n {\n LOG(INFO) << \"Server ThreadCount: \" << numThreadsInPool;\n }\n\n ThreadPool& AcquireThreadPool() { return m_ThreadPool; }\n\n private:\n ThreadPool m_ThreadPool;\n};\n\n// Contexts hold the state and provide the definition of the work to be performed by the RPC.\n// This is where you define what gets executed for a given RPC.\n// Incoming Message = simple::Input (RequestType)\n// Outgoing Message = simple::Output (ResponseType)\nclass SimpleContext final : public Context\n{\n void ExecuteRPC(RequestType& input, ResponseType& output) final override\n {\n // We could do work here, but we'd block the TPS, i.e. the threads pulling messages\n // off the incoming recieve queue. Very quick responses are best done here; however,\n // longer running workload should be offloaded so the TPS can avoid being blocked.\n GetResources()->AcquireThreadPool().enqueue([this, &input, &output] {\n // Now running on a worker thread of the ThreadPool defined in SimpleResources.\n // Here we are just echoing back the incoming // batch_id; however, in later\n // examples, we'll show how to run an async cuda pipline.\n LOG_FIRST_N(INFO, 20) << \"Tag = \" << Tag()\n << \" Thread = \" << std::this_thread::get_id();\n output.set_batch_id(input.batch_id());\n this->FinishResponse();\n });\n // The TPS thread is now free to continue processing message - async ftw!\n }\n};\n\nint main(int argc, char* argv[])\n{\n FLAGS_alsologtostderr = 1; // Log to console\n\n ::google::InitGoogleLogging(\"simpleServer\");\n ::google::ParseCommandLineFlags(&argc, &argv, true);\n\n // A server will bind an IP:PORT to listen on\n Server server(\"0.0.0.0:50051\");\n\n // A server can host multiple services\n LOG(INFO) << \"Register Service (simple::Inference) with Server\";\n auto simpleInference = server.RegisterAsyncService();\n\n // An RPC has two components that need to be specified when registering with the service:\n // 1) Type of Execution Context (SimpleContext). The execution context defines the behavor\n // of the RPC, i.e. it contains the control logic for the execution of the RPC.\n // 2) The Request function (RequestCompute) which was generated by gRPC when compiling the\n // protobuf which defined the service. This function is responsible for queuing the\n // RPC's execution context to the\n LOG(INFO) << \"Register RPC (simple::Inference::Compute) with Service (simple::Inference)\";\n auto rpcCompute = simpleInference->RegisterRPC(\n &simple::Inference::AsyncService::RequestCompute);\n\n LOG(INFO) << \"Initializing Resources for RPC (simple::Inference::Compute)\";\n auto rpcResources = std::make_shared(FLAGS_thread_count);\n\n // Create Executors - Executors provide the messaging processing resources for the RPCs\n // Multiple Executors can be registered with a Server. The executor is responsible\n // for pulling incoming message off the receive queue and executing the associated\n // context. By default, an executor only uses a single thread. A typical usecase is\n // an Executor executes a context, which immediate pushes the work to a thread pool.\n // However, for very low-latency messaging, you might want to use a multi-threaded\n // Executor and a Blocking Context - meaning the Context performs the entire RPC function\n // on the Executor's thread.\n LOG(INFO) << \"Creating Executor\";\n auto executor = server.RegisterExecutor(new Executor(1));\n\n // You can register RPC execution contexts from any registered RPC on any executor.\n // The power of that will become clear in later examples. For now, we will register\n // 10 instances of the simple::Inference::Compute RPC's SimpleContext execution context\n // with the Executor.\n LOG(INFO) << \"Creating Execution Contexts for RPC (simple::Inference::Compute) with Executor\";\n executor->RegisterContexts(rpcCompute, rpcResources, 10);\n\n LOG(INFO) << \"Running Server\";\n server.Run(std::chrono::milliseconds(2000), [] {\n // This is a timeout loop executed every 2seconds\n // Run() with no arguments will run an empty timeout loop every 5 seconds.\n // RunAsync() will return immediately, its your responsibility to ensure the\n // server doesn't go out of scope or a Shutdown will be triggered on your services.\n });\n}\n" }, { "path": "jupyter_notebook_config.py", "content": "# Configuration file for jupyter-notebook.\n\n#------------------------------------------------------------------------------\n# Application(SingletonConfigurable) configuration\n#------------------------------------------------------------------------------\n\n## This is an application.\n\n## The date format used by logging formatters for %(asctime)s\n#c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S'\n\n## The Logging format template\n#c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s'\n\n## Set the log level by value or name.\n#c.Application.log_level = 30\n\n#------------------------------------------------------------------------------\n# JupyterApp(Application) configuration\n#------------------------------------------------------------------------------\n\n## Base class for Jupyter applications\n\n## Answer yes to any prompts.\n#c.JupyterApp.answer_yes = False\n\n## Full path of a config file.\n#c.JupyterApp.config_file = ''\n\n## Specify a config file to load.\n#c.JupyterApp.config_file_name = ''\n\n## Generate default config file.\n#c.JupyterApp.generate_config = False\n\n#------------------------------------------------------------------------------\n# NotebookApp(JupyterApp) configuration\n#------------------------------------------------------------------------------\n\n## Set the Access-Control-Allow-Credentials: true header\n#c.NotebookApp.allow_credentials = False\n\n## Set the Access-Control-Allow-Origin header\n# \n# Use '*' to allow any origin to access your server.\n# \n# Takes precedence over allow_origin_pat.\n#c.NotebookApp.allow_origin = ''\n\n## Use a regular expression for the Access-Control-Allow-Origin header\n# \n# Requests from an origin matching the expression will get replies with:\n# \n# Access-Control-Allow-Origin: origin\n# \n# where `origin` is the origin of the request.\n# \n# Ignored if allow_origin is set.\n#c.NotebookApp.allow_origin_pat = ''\n\n## Allow password to be changed at login for the notebook server.\n# \n# While loggin in with a token, the notebook server UI will give the opportunity\n# to the user to enter a new password at the same time that will replace the\n# token login mechanism.\n# \n# This can be set to false to prevent changing password from the UI/API.\n#c.NotebookApp.allow_password_change = True\n\n## Allow requests where the Host header doesn't point to a local server\n# \n# By default, requests get a 403 forbidden response if the 'Host' header shows\n# that the browser thinks it's on a non-local domain. Setting this option to\n# True disables this check.\n# \n# This protects against 'DNS rebinding' attacks, where a remote web server\n# serves you a page and then changes its DNS to send later requests to a local\n# IP, bypassing same-origin checks.\n# \n# Local IP addresses (such as 127.0.0.1 and ::1) are allowed as local, along\n# with hostnames configured in local_hostnames.\nc.NotebookApp.allow_remote_access = True\n\n## Whether to allow the user to run the notebook as root.\nc.NotebookApp.allow_root = True\n\n## DEPRECATED use base_url\n#c.NotebookApp.base_project_url = '/'\n\n## The base URL for the notebook server.\n# \n# Leading and trailing slashes can be omitted, and will automatically be added.\n#c.NotebookApp.base_url = '/'\n\n## Specify what command to use to invoke a web browser when opening the notebook.\n# If not specified, the default browser will be determined by the `webbrowser`\n# standard library module, which allows setting of the BROWSER environment\n# variable to override it.\n#c.NotebookApp.browser = ''\n\n## The full path to an SSL/TLS certificate file.\n#c.NotebookApp.certfile = ''\n\n## The full path to a certificate authority certificate for SSL/TLS client\n# authentication.\n#c.NotebookApp.client_ca = ''\n\n## The config manager class to use\n#c.NotebookApp.config_manager_class = 'notebook.services.config.manager.ConfigManager'\n\n## The notebook manager class to use.\n#c.NotebookApp.contents_manager_class = 'notebook.services.contents.largefilemanager.LargeFileManager'\n\n## Extra keyword arguments to pass to `set_secure_cookie`. See tornado's\n# set_secure_cookie docs for details.\n#c.NotebookApp.cookie_options = {}\n\n## The random bytes used to secure cookies. By default this is a new random\n# number every time you start the Notebook. Set it to a value in a config file\n# to enable logins to persist across server sessions.\n# \n# Note: Cookie secrets should be kept private, do not share config files with\n# cookie_secret stored in plaintext (you can read the value from a file).\n#c.NotebookApp.cookie_secret = b''\n\n## The file where the cookie secret is stored.\n#c.NotebookApp.cookie_secret_file = ''\n\n## Override URL shown to users.\n# \n# Replace actual URL, including protocol, address, port and base URL, with the\n# given value when displaying URL to the users. Do not change the actual\n# connection URL. If authentication token is enabled, the token is added to the\n# custom URL automatically.\n# \n# This option is intended to be used when the URL to display to the user cannot\n# be determined reliably by the Jupyter notebook server (proxified or\n# containerized setups for example).\n#c.NotebookApp.custom_display_url = ''\n\n## The default URL to redirect to from `/`\n#c.NotebookApp.default_url = '/tree'\n\n## Disable cross-site-request-forgery protection\n# \n# Jupyter notebook 4.3.1 introduces protection from cross-site request\n# forgeries, requiring API requests to either:\n# \n# - originate from pages served by this server (validated with XSRF cookie and\n# token), or - authenticate with a token\n# \n# Some anonymous compute resources still desire the ability to run code,\n# completely without authentication. These services can disable all\n# authentication and security checks, with the full knowledge of what that\n# implies.\n#c.NotebookApp.disable_check_xsrf = False\n\n## Whether to enable MathJax for typesetting math/TeX\n# \n# MathJax is the javascript library Jupyter uses to render math/LaTeX. It is\n# very large, so you may want to disable it if you have a slow internet\n# connection, or for offline use of the notebook.\n# \n# When disabled, equations etc. will appear as their untransformed TeX source.\n#c.NotebookApp.enable_mathjax = True\n\n## extra paths to look for Javascript notebook extensions\n#c.NotebookApp.extra_nbextensions_path = []\n\n## handlers that should be loaded at higher priority than the default services\n#c.NotebookApp.extra_services = []\n\n## Extra paths to search for serving static files.\n# \n# This allows adding javascript/css to be available from the notebook server\n# machine, or overriding individual files in the IPython\n#c.NotebookApp.extra_static_paths = []\n\n## Extra paths to search for serving jinja templates.\n# \n# Can be used to override templates from notebook.templates.\n#c.NotebookApp.extra_template_paths = []\n\n## \n#c.NotebookApp.file_to_run = ''\n\n## Extra keyword arguments to pass to `get_secure_cookie`. See tornado's\n# get_secure_cookie docs for details.\n#c.NotebookApp.get_secure_cookie_kwargs = {}\n\n## Deprecated: Use minified JS file or not, mainly use during dev to avoid JS\n# recompilation\n#c.NotebookApp.ignore_minified_js = False\n\n## (bytes/sec) Maximum rate at which stream output can be sent on iopub before\n# they are limited.\n#c.NotebookApp.iopub_data_rate_limit = 1000000\n\n## (msgs/sec) Maximum rate at which messages can be sent on iopub before they are\n# limited.\n#c.NotebookApp.iopub_msg_rate_limit = 1000\n\n## The IP address the notebook server will listen on.\nc.NotebookApp.ip = '0.0.0.0'\n\n## Supply extra arguments that will be passed to Jinja environment.\n#c.NotebookApp.jinja_environment_options = {}\n\n## Extra variables to supply to jinja templates when rendering.\n#c.NotebookApp.jinja_template_vars = {}\n\n## The kernel manager class to use.\n#c.NotebookApp.kernel_manager_class = 'notebook.services.kernels.kernelmanager.MappingKernelManager'\n\n## The kernel spec manager class to use. Should be a subclass of\n# `jupyter_client.kernelspec.KernelSpecManager`.\n# \n# The Api of KernelSpecManager is provisional and might change without warning\n# between this version of Jupyter and the next stable one.\n#c.NotebookApp.kernel_spec_manager_class = 'jupyter_client.kernelspec.KernelSpecManager'\n\n## The full path to a private key file for usage with SSL/TLS.\n#c.NotebookApp.keyfile = ''\n\n## Hostnames to allow as local when allow_remote_access is False.\n# \n# Local IP addresses (such as 127.0.0.1 and ::1) are automatically accepted as\n# local as well.\n#c.NotebookApp.local_hostnames = ['localhost']\n\n## The login handler class to use.\n#c.NotebookApp.login_handler_class = 'notebook.auth.login.LoginHandler'\n\n## The logout handler class to use.\n#c.NotebookApp.logout_handler_class = 'notebook.auth.logout.LogoutHandler'\n\n## The MathJax.js configuration file that is to be used.\n#c.NotebookApp.mathjax_config = 'TeX-AMS-MML_HTMLorMML-full,Safe'\n\n## A custom url for MathJax.js. Should be in the form of a case-sensitive url to\n# MathJax, for example: /static/components/MathJax/MathJax.js\n#c.NotebookApp.mathjax_url = ''\n\n## Sets the maximum allowed size of the client request body, specified in the\n# Content-Length request header field. If the size in a request exceeds the\n# configured value, a malformed HTTP message is returned to the client.\n# \n# Note: max_body_size is applied even in streaming mode.\n#c.NotebookApp.max_body_size = 536870912\n\n## Gets or sets the maximum amount of memory, in bytes, that is allocated for\n# use by the buffer manager.\n#c.NotebookApp.max_buffer_size = 536870912\n\n## Dict of Python modules to load as notebook server extensions.Entry values can\n# be used to enable and disable the loading ofthe extensions. The extensions\n# will be loaded in alphabetical order.\n#c.NotebookApp.nbserver_extensions = {}\n\n## The directory to use for notebooks and kernels.\nc.NotebookApp.notebook_dir = '/work'\n\n## Whether to open in a browser after starting. The specific browser used is\n# platform dependent and determined by the python standard library `webbrowser`\n# module, unless it is overridden using the --browser (NotebookApp.browser)\n# configuration option.\n#c.NotebookApp.open_browser = True\n\n## Hashed password to use for web authentication.\n# \n# To generate, type in a python/IPython shell:\n# \n# from notebook.auth import passwd; passwd()\n# \n# The string should be of the form type:salt:hashed-password.\n#c.NotebookApp.password = ''\n\n## Forces users to use a password for the Notebook server. This is useful in a\n# multi user environment, for instance when everybody in the LAN can access each\n# other's machine through ssh.\n# \n# In such a case, server the notebook server on localhost is not secure since\n# any user can connect to the notebook server via ssh.\n#c.NotebookApp.password_required = False\n\n## The port the notebook server will listen on.\n#c.NotebookApp.port = 8888\n\n## The number of additional ports to try if the specified port is not available.\n#c.NotebookApp.port_retries = 50\n\n## DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.\n#c.NotebookApp.pylab = 'disabled'\n\n## If True, display a button in the dashboard to quit (shutdown the notebook\n# server).\n#c.NotebookApp.quit_button = True\n\n## (sec) Time window used to check the message and data rate limits.\n#c.NotebookApp.rate_limit_window = 3\n\n## Reraise exceptions encountered loading server extensions?\n#c.NotebookApp.reraise_server_extension_failures = False\n\n## DEPRECATED use the nbserver_extensions dict instead\n#c.NotebookApp.server_extensions = []\n\n## The session manager class to use.\n#c.NotebookApp.session_manager_class = 'notebook.services.sessions.sessionmanager.SessionManager'\n\n## Shut down the server after N seconds with no kernels or terminals running and\n# no activity. This can be used together with culling idle kernels\n# (MappingKernelManager.cull_idle_timeout) to shutdown the notebook server when\n# it's not in use. This is not precisely timed: it may shut down up to a minute\n# later. 0 (the default) disables this automatic shutdown.\n#c.NotebookApp.shutdown_no_activity_timeout = 0\n\n## Supply SSL options for the tornado HTTPServer. See the tornado docs for\n# details.\n#c.NotebookApp.ssl_options = {}\n\n## Supply overrides for terminado. Currently only supports \"shell_command\".\n#c.NotebookApp.terminado_settings = {}\n\n## Set to False to disable terminals.\n# \n# This does *not* make the notebook server more secure by itself. Anything the\n# user can in a terminal, they can also do in a notebook.\n# \n# Terminals may also be automatically disabled if the terminado package is not\n# available.\n#c.NotebookApp.terminals_enabled = True\n\n## Token used for authenticating first-time connections to the server.\n# \n# When no password is enabled, the default is to generate a new, random token.\n# \n# Setting to an empty string disables authentication altogether, which is NOT\n# RECOMMENDED.\n#c.NotebookApp.token = ''\n\n## Supply overrides for the tornado.web.Application that the Jupyter notebook\n# uses.\n#c.NotebookApp.tornado_settings = {}\n\n## Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-\n# For headerssent by the upstream reverse proxy. Necessary if the proxy handles\n# SSL\n#c.NotebookApp.trust_xheaders = False\n\n## DEPRECATED, use tornado_settings\n#c.NotebookApp.webapp_settings = {}\n\n## Specify Where to open the notebook on startup. This is the `new` argument\n# passed to the standard library method `webbrowser.open`. The behaviour is not\n# guaranteed, but depends on browser support. Valid values are:\n# \n# - 2 opens a new tab,\n# - 1 opens a new window,\n# - 0 opens in an existing window.\n# \n# See the `webbrowser.open` documentation for details.\n#c.NotebookApp.webbrowser_open_new = 2\n\n## Set the tornado compression options for websocket connections.\n# \n# This value will be returned from\n# :meth:`WebSocketHandler.get_compression_options`. None (default) will disable\n# compression. A dict (even an empty one) will enable compression.\n# \n# See the tornado docs for WebSocketHandler.get_compression_options for details.\n#c.NotebookApp.websocket_compression_options = None\n\n## The base URL for websockets, if it differs from the HTTP server (hint: it\n# almost certainly doesn't).\n# \n# Should be in the form of an HTTP origin: ws[s]://hostname[:port]\n#c.NotebookApp.websocket_url = ''\n\n#------------------------------------------------------------------------------\n# ConnectionFileMixin(LoggingConfigurable) configuration\n#------------------------------------------------------------------------------\n\n## Mixin for configurable classes that work with connection files\n\n## JSON file in which to store connection info [default: kernel-.json]\n# \n# This file will contain the IP, ports, and authentication key needed to connect\n# clients to this kernel. By default, this file will be created in the security\n# dir of the current profile, but can be specified by absolute path.\n#c.ConnectionFileMixin.connection_file = ''\n\n## set the control (ROUTER) port [default: random]\n#c.ConnectionFileMixin.control_port = 0\n\n## set the heartbeat port [default: random]\n#c.ConnectionFileMixin.hb_port = 0\n\n## set the iopub (PUB) port [default: random]\n#c.ConnectionFileMixin.iopub_port = 0\n\n## Set the kernel's IP address [default localhost]. If the IP address is\n# something other than localhost, then Consoles on other machines will be able\n# to connect to the Kernel, so be careful!\n#c.ConnectionFileMixin.ip = ''\n\n## set the shell (ROUTER) port [default: random]\n#c.ConnectionFileMixin.shell_port = 0\n\n## set the stdin (ROUTER) port [default: random]\n#c.ConnectionFileMixin.stdin_port = 0\n\n## \n#c.ConnectionFileMixin.transport = 'tcp'\n\n#------------------------------------------------------------------------------\n# KernelManager(ConnectionFileMixin) configuration\n#------------------------------------------------------------------------------\n\n## Manages a single kernel in a subprocess on this host.\n# \n# This version starts kernels with Popen.\n\n## Should we autorestart the kernel if it dies.\n#c.KernelManager.autorestart = True\n\n## DEPRECATED: Use kernel_name instead.\n# \n# The Popen Command to launch the kernel. Override this if you have a custom\n# kernel. If kernel_cmd is specified in a configuration file, Jupyter does not\n# pass any arguments to the kernel, because it cannot make any assumptions about\n# the arguments that the kernel understands. In particular, this means that the\n# kernel does not receive the option --debug if it given on the Jupyter command\n# line.\n#c.KernelManager.kernel_cmd = []\n\n## Time to wait for a kernel to terminate before killing it, in seconds.\n#c.KernelManager.shutdown_wait_time = 5.0\n\n#------------------------------------------------------------------------------\n# Session(Configurable) configuration\n#------------------------------------------------------------------------------\n\n## Object for handling serialization and sending of messages.\n# \n# The Session object handles building messages and sending them with ZMQ sockets\n# or ZMQStream objects. Objects can communicate with each other over the\n# network via Session objects, and only need to work with the dict-based IPython\n# message spec. The Session will handle serialization/deserialization, security,\n# and metadata.\n# \n# Sessions support configurable serialization via packer/unpacker traits, and\n# signing with HMAC digests via the key/keyfile traits.\n# \n# Parameters ----------\n# \n# debug : bool\n# whether to trigger extra debugging statements\n# packer/unpacker : str : 'json', 'pickle' or import_string\n# importstrings for methods to serialize message parts. If just\n# 'json' or 'pickle', predefined JSON and pickle packers will be used.\n# Otherwise, the entire importstring must be used.\n# \n# The functions must accept at least valid JSON input, and output *bytes*.\n# \n# For example, to use msgpack:\n# packer = 'msgpack.packb', unpacker='msgpack.unpackb'\n# pack/unpack : callables\n# You can also set the pack/unpack callables for serialization directly.\n# session : bytes\n# the ID of this Session object. The default is to generate a new UUID.\n# username : unicode\n# username added to message headers. The default is to ask the OS.\n# key : bytes\n# The key used to initialize an HMAC signature. If unset, messages\n# will not be signed or checked.\n# keyfile : filepath\n# The file containing a key. If this is set, `key` will be initialized\n# to the contents of the file.\n\n## Threshold (in bytes) beyond which an object's buffer should be extracted to\n# avoid pickling.\n#c.Session.buffer_threshold = 1024\n\n## Whether to check PID to protect against calls after fork.\n# \n# This check can be disabled if fork-safety is handled elsewhere.\n#c.Session.check_pid = True\n\n## Threshold (in bytes) beyond which a buffer should be sent without copying.\n#c.Session.copy_threshold = 65536\n\n## Debug output in the Session\n#c.Session.debug = False\n\n## The maximum number of digests to remember.\n# \n# The digest history will be culled when it exceeds this value.\n#c.Session.digest_history_size = 65536\n\n## The maximum number of items for a container to be introspected for custom\n# serialization. Containers larger than this are pickled outright.\n#c.Session.item_threshold = 64\n\n## execution key, for signing messages.\n#c.Session.key = b''\n\n## path to file containing execution key.\n#c.Session.keyfile = ''\n\n## Metadata dictionary, which serves as the default top-level metadata dict for\n# each message.\n#c.Session.metadata = {}\n\n## The name of the packer for serializing messages. Should be one of 'json',\n# 'pickle', or an import name for a custom callable serializer.\n#c.Session.packer = 'json'\n\n## The UUID identifying this session.\n#c.Session.session = ''\n\n## The digest scheme used to construct the message signatures. Must have the form\n# 'hmac-HASH'.\n#c.Session.signature_scheme = 'hmac-sha256'\n\n## The name of the unpacker for unserializing messages. Only used with custom\n# functions for `packer`.\n#c.Session.unpacker = 'json'\n\n## Username for the Session. Default is your system username.\n#c.Session.username = 'username'\n\n#------------------------------------------------------------------------------\n# MultiKernelManager(LoggingConfigurable) configuration\n#------------------------------------------------------------------------------\n\n## A class for managing multiple kernels.\n\n## The name of the default kernel to start\n#c.MultiKernelManager.default_kernel_name = 'python3'\n\n## The kernel manager class. This is configurable to allow subclassing of the\n# KernelManager for customized behavior.\n#c.MultiKernelManager.kernel_manager_class = 'jupyter_client.ioloop.IOLoopKernelManager'\n\n#------------------------------------------------------------------------------\n# MappingKernelManager(MultiKernelManager) configuration\n#------------------------------------------------------------------------------\n\n## A KernelManager that handles notebook mapping and HTTP error handling\n\n## Whether messages from kernels whose frontends have disconnected should be\n# buffered in-memory.\n# \n# When True (default), messages are buffered and replayed on reconnect, avoiding\n# lost messages due to interrupted connectivity.\n# \n# Disable if long-running kernels will produce too much output while no\n# frontends are connected.\n#c.MappingKernelManager.buffer_offline_messages = True\n\n## Whether to consider culling kernels which are busy. Only effective if\n# cull_idle_timeout > 0.\n#c.MappingKernelManager.cull_busy = False\n\n## Whether to consider culling kernels which have one or more connections. Only\n# effective if cull_idle_timeout > 0.\n#c.MappingKernelManager.cull_connected = False\n\n## Timeout (in seconds) after which a kernel is considered idle and ready to be\n# culled. Values of 0 or lower disable culling. Very short timeouts may result\n# in kernels being culled for users with poor network connections.\n#c.MappingKernelManager.cull_idle_timeout = 0\n\n## The interval (in seconds) on which to check for idle kernels exceeding the\n# cull timeout value.\n#c.MappingKernelManager.cull_interval = 300\n\n## Timeout for giving up on a kernel (in seconds).\n# \n# On starting and restarting kernels, we check whether the kernel is running and\n# responsive by sending kernel_info_requests. This sets the timeout in seconds\n# for how long the kernel can take before being presumed dead. This affects the\n# MappingKernelManager (which handles kernel restarts) and the\n# ZMQChannelsHandler (which handles the startup).\n#c.MappingKernelManager.kernel_info_timeout = 60\n\n## \n#c.MappingKernelManager.root_dir = ''\n\n#------------------------------------------------------------------------------\n# ContentsManager(LoggingConfigurable) configuration\n#------------------------------------------------------------------------------\n\n## Base class for serving files and directories.\n# \n# This serves any text or binary file, as well as directories, with special\n# handling for JSON notebook documents.\n# \n# Most APIs take a path argument, which is always an API-style unicode path, and\n# always refers to a directory.\n# \n# - unicode, not url-escaped\n# - '/'-separated\n# - leading and trailing '/' will be stripped\n# - if unspecified, path defaults to '',\n# indicating the root path.\n\n## Allow access to hidden files\n#c.ContentsManager.allow_hidden = False\n\n## \n#c.ContentsManager.checkpoints = None\n\n## \n#c.ContentsManager.checkpoints_class = 'notebook.services.contents.checkpoints.Checkpoints'\n\n## \n#c.ContentsManager.checkpoints_kwargs = {}\n\n## handler class to use when serving raw file requests.\n# \n# Default is a fallback that talks to the ContentsManager API, which may be\n# inefficient, especially for large files.\n# \n# Local files-based ContentsManagers can use a StaticFileHandler subclass, which\n# will be much more efficient.\n# \n# Access to these files should be Authenticated.\n#c.ContentsManager.files_handler_class = 'notebook.files.handlers.FilesHandler'\n\n## Extra parameters to pass to files_handler_class.\n# \n# For example, StaticFileHandlers generally expect a `path` argument specifying\n# the root directory from which to serve files.\n#c.ContentsManager.files_handler_params = {}\n\n## Glob patterns to hide in file and directory listings.\n#c.ContentsManager.hide_globs = ['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dylib', '*~']\n\n## Python callable or importstring thereof\n# \n# To be called on a contents model prior to save.\n# \n# This can be used to process the structure, such as removing notebook outputs\n# or other side effects that should not be saved.\n# \n# It will be called as (all arguments passed by keyword)::\n# \n# hook(path=path, model=model, contents_manager=self)\n# \n# - model: the model to be saved. Includes file contents.\n# Modifying this dict will affect the file that is stored.\n# - path: the API path of the save destination\n# - contents_manager: this ContentsManager instance\n#c.ContentsManager.pre_save_hook = None\n\n## \n#c.ContentsManager.root_dir = '/'\n\n## The base name used when creating untitled directories.\n#c.ContentsManager.untitled_directory = 'Untitled Folder'\n\n## The base name used when creating untitled files.\n#c.ContentsManager.untitled_file = 'untitled'\n\n## The base name used when creating untitled notebooks.\n#c.ContentsManager.untitled_notebook = 'Untitled'\n\n#------------------------------------------------------------------------------\n# FileManagerMixin(Configurable) configuration\n#------------------------------------------------------------------------------\n\n## Mixin for ContentsAPI classes that interact with the filesystem.\n# \n# Provides facilities for reading, writing, and copying both notebooks and\n# generic files.\n# \n# Shared by FileContentsManager and FileCheckpoints.\n# \n# Note ---- Classes using this mixin must provide the following attributes:\n# \n# root_dir : unicode\n# A directory against against which API-style paths are to be resolved.\n# \n# log : logging.Logger\n\n## By default notebooks are saved on disk on a temporary file and then if\n# succefully written, it replaces the old ones. This procedure, namely\n# 'atomic_writing', causes some bugs on file system whitout operation order\n# enforcement (like some networked fs). If set to False, the new notebook is\n# written directly on the old one which could fail (eg: full filesystem or quota\n# )\n#c.FileManagerMixin.use_atomic_writing = True\n\n#------------------------------------------------------------------------------\n# FileContentsManager(FileManagerMixin,ContentsManager) configuration\n#------------------------------------------------------------------------------\n\n## If True (default), deleting files will send them to the platform's\n# trash/recycle bin, where they can be recovered. If False, deleting files\n# really deletes them.\n#c.FileContentsManager.delete_to_trash = True\n\n## Python callable or importstring thereof\n# \n# to be called on the path of a file just saved.\n# \n# This can be used to process the file on disk, such as converting the notebook\n# to a script or HTML via nbconvert.\n# \n# It will be called as (all arguments passed by keyword)::\n# \n# hook(os_path=os_path, model=model, contents_manager=instance)\n# \n# - path: the filesystem path to the file just written - model: the model\n# representing the file - contents_manager: this ContentsManager instance\n#c.FileContentsManager.post_save_hook = None\n\n## \n#c.FileContentsManager.root_dir = ''\n\n## DEPRECATED, use post_save_hook. Will be removed in Notebook 5.0\n#c.FileContentsManager.save_script = False\n\n#------------------------------------------------------------------------------\n# NotebookNotary(LoggingConfigurable) configuration\n#------------------------------------------------------------------------------\n\n## A class for computing and verifying notebook signatures.\n\n## The hashing algorithm used to sign notebooks.\n#c.NotebookNotary.algorithm = 'sha256'\n\n## The sqlite file in which to store notebook signatures. By default, this will\n# be in your Jupyter data directory. You can set it to ':memory:' to disable\n# sqlite writing to the filesystem.\n#c.NotebookNotary.db_file = ''\n\n## The secret key with which notebooks are signed.\n#c.NotebookNotary.secret = b''\n\n## The file where the secret key is stored.\n#c.NotebookNotary.secret_file = ''\n\n## A callable returning the storage backend for notebook signatures. The default\n# uses an SQLite database.\n#c.NotebookNotary.store_factory = traitlets.Undefined\n\n#------------------------------------------------------------------------------\n# KernelSpecManager(LoggingConfigurable) configuration\n#------------------------------------------------------------------------------\n\n## If there is no Python kernelspec registered and the IPython kernel is\n# available, ensure it is added to the spec list.\n#c.KernelSpecManager.ensure_native_kernel = True\n\n## The kernel spec class. This is configurable to allow subclassing of the\n# KernelSpecManager for customized behavior.\n#c.KernelSpecManager.kernel_spec_class = 'jupyter_client.kernelspec.KernelSpec'\n\n## Whitelist of allowed kernel names.\n# \n# By default, all installed kernels are allowed.\n#c.KernelSpecManager.whitelist = set()\n" }, { "path": "models/README.md", "content": "## Sample Models\n\nIncluded in this folder are a collection of open source models and\nsome scripts to build TensorRT engines from these models.\n\nCurrently, the samples provided only generate TensorRT engines\nwith random weights and are only good for synthetic tests.\n\nTODO: update scripts to pull open sourced weights for fully functional\nmodels.\n\n\n## Credits\n\nCaffe ResNet-50 and ResNet-152 models from [KaimingHe/deep-residual-networks]\n(https://github.com/KaimingHe/deep-residual-networks) are included without modification.\n\n> The MIT License (MIT)\n> \n> Copyright (c) 2016 Shaoqing Ren\n> \n> Permission is hereby granted, free of charge, to any person obtaining a copy\n> of this software and associated documentation files (the \"Software\"), to deal\n> in the Software without restriction, including without limitation the rights\n> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n> copies of the Software, and to permit persons to whom the Software is\n> furnished to do so, subject to the following conditions:\n> \n> The above copyright notice and this permission notice shall be included in all\n> copies or substantial portions of the Software.\n> \n> THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n> SOFTWARE.\n" }, { "path": "models/ResNet-152-deploy.prototxt", "content": "name: \"ResNet-152\"\ninput: \"data\"\ninput_dim: 1\ninput_dim: 3\ninput_dim: 224\ninput_dim: 224\n\nlayer {\n\tbottom: \"data\"\n\ttop: \"conv1\"\n\tname: \"conv1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 7\n\t\tpad: 3\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"conv1\"\n\ttop: \"conv1\"\n\tname: \"bn_conv1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"conv1\"\n\ttop: \"conv1\"\n\tname: \"scale_conv1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"conv1\"\n\tbottom: \"conv1\"\n\tname: \"conv1_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"conv1\"\n\ttop: \"pool1\"\n\tname: \"pool1\"\n\ttype: \"Pooling\"\n\tpooling_param {\n\t\tkernel_size: 3\n\t\tstride: 2\n\t\tpool: MAX\n\t}\n}\n\nlayer {\n\tbottom: \"pool1\"\n\ttop: \"res2a_branch1\"\n\tname: \"res2a_branch1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch1\"\n\ttop: \"res2a_branch1\"\n\tname: \"bn2a_branch1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch1\"\n\ttop: \"res2a_branch1\"\n\tname: \"scale2a_branch1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"pool1\"\n\ttop: \"res2a_branch2a\"\n\tname: \"res2a_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2a\"\n\ttop: \"res2a_branch2a\"\n\tname: \"bn2a_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2a\"\n\ttop: \"res2a_branch2a\"\n\tname: \"scale2a_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res2a_branch2a\"\n\tbottom: \"res2a_branch2a\"\n\tname: \"res2a_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2a_branch2a\"\n\ttop: \"res2a_branch2b\"\n\tname: \"res2a_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2b\"\n\ttop: \"res2a_branch2b\"\n\tname: \"bn2a_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2b\"\n\ttop: \"res2a_branch2b\"\n\tname: \"scale2a_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res2a_branch2b\"\n\tbottom: \"res2a_branch2b\"\n\tname: \"res2a_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2a_branch2b\"\n\ttop: \"res2a_branch2c\"\n\tname: \"res2a_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2c\"\n\ttop: \"res2a_branch2c\"\n\tname: \"bn2a_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2c\"\n\ttop: \"res2a_branch2c\"\n\tname: \"scale2a_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch1\"\n\tbottom: \"res2a_branch2c\"\n\ttop: \"res2a\"\n\tname: \"res2a\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res2a\"\n\ttop: \"res2a\"\n\tname: \"res2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2a\"\n\ttop: \"res2b_branch2a\"\n\tname: \"res2b_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2a\"\n\ttop: \"res2b_branch2a\"\n\tname: \"bn2b_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2a\"\n\ttop: \"res2b_branch2a\"\n\tname: \"scale2b_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res2b_branch2a\"\n\tbottom: \"res2b_branch2a\"\n\tname: \"res2b_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2b_branch2a\"\n\ttop: \"res2b_branch2b\"\n\tname: \"res2b_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2b\"\n\ttop: \"res2b_branch2b\"\n\tname: \"bn2b_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2b\"\n\ttop: \"res2b_branch2b\"\n\tname: \"scale2b_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res2b_branch2b\"\n\tbottom: \"res2b_branch2b\"\n\tname: \"res2b_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2b_branch2b\"\n\ttop: \"res2b_branch2c\"\n\tname: \"res2b_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2c\"\n\ttop: \"res2b_branch2c\"\n\tname: \"bn2b_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2c\"\n\ttop: \"res2b_branch2c\"\n\tname: \"scale2b_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a\"\n\tbottom: \"res2b_branch2c\"\n\ttop: \"res2b\"\n\tname: \"res2b\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res2b\"\n\ttop: \"res2b\"\n\tname: \"res2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2b\"\n\ttop: \"res2c_branch2a\"\n\tname: \"res2c_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2a\"\n\ttop: \"res2c_branch2a\"\n\tname: \"bn2c_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2a\"\n\ttop: \"res2c_branch2a\"\n\tname: \"scale2c_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res2c_branch2a\"\n\tbottom: \"res2c_branch2a\"\n\tname: \"res2c_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2c_branch2a\"\n\ttop: \"res2c_branch2b\"\n\tname: \"res2c_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2b\"\n\ttop: \"res2c_branch2b\"\n\tname: \"bn2c_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2b\"\n\ttop: \"res2c_branch2b\"\n\tname: \"scale2c_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res2c_branch2b\"\n\tbottom: \"res2c_branch2b\"\n\tname: \"res2c_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2c_branch2b\"\n\ttop: \"res2c_branch2c\"\n\tname: \"res2c_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2c\"\n\ttop: \"res2c_branch2c\"\n\tname: \"bn2c_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2c\"\n\ttop: \"res2c_branch2c\"\n\tname: \"scale2c_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b\"\n\tbottom: \"res2c_branch2c\"\n\ttop: \"res2c\"\n\tname: \"res2c\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res2c\"\n\ttop: \"res2c\"\n\tname: \"res2c_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2c\"\n\ttop: \"res3a_branch1\"\n\tname: \"res3a_branch1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch1\"\n\ttop: \"res3a_branch1\"\n\tname: \"bn3a_branch1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch1\"\n\ttop: \"res3a_branch1\"\n\tname: \"scale3a_branch1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c\"\n\ttop: \"res3a_branch2a\"\n\tname: \"res3a_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2a\"\n\ttop: \"res3a_branch2a\"\n\tname: \"bn3a_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2a\"\n\ttop: \"res3a_branch2a\"\n\tname: \"scale3a_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3a_branch2a\"\n\tbottom: \"res3a_branch2a\"\n\tname: \"res3a_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3a_branch2a\"\n\ttop: \"res3a_branch2b\"\n\tname: \"res3a_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2b\"\n\ttop: \"res3a_branch2b\"\n\tname: \"bn3a_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2b\"\n\ttop: \"res3a_branch2b\"\n\tname: \"scale3a_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3a_branch2b\"\n\tbottom: \"res3a_branch2b\"\n\tname: \"res3a_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3a_branch2b\"\n\ttop: \"res3a_branch2c\"\n\tname: \"res3a_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2c\"\n\ttop: \"res3a_branch2c\"\n\tname: \"bn3a_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2c\"\n\ttop: \"res3a_branch2c\"\n\tname: \"scale3a_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch1\"\n\tbottom: \"res3a_branch2c\"\n\ttop: \"res3a\"\n\tname: \"res3a\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3a\"\n\ttop: \"res3a\"\n\tname: \"res3a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3a\"\n\ttop: \"res3b1_branch2a\"\n\tname: \"res3b1_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b1_branch2a\"\n\ttop: \"res3b1_branch2a\"\n\tname: \"bn3b1_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b1_branch2a\"\n\ttop: \"res3b1_branch2a\"\n\tname: \"scale3b1_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b1_branch2a\"\n\tbottom: \"res3b1_branch2a\"\n\tname: \"res3b1_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b1_branch2a\"\n\ttop: \"res3b1_branch2b\"\n\tname: \"res3b1_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b1_branch2b\"\n\ttop: \"res3b1_branch2b\"\n\tname: \"bn3b1_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b1_branch2b\"\n\ttop: \"res3b1_branch2b\"\n\tname: \"scale3b1_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b1_branch2b\"\n\tbottom: \"res3b1_branch2b\"\n\tname: \"res3b1_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b1_branch2b\"\n\ttop: \"res3b1_branch2c\"\n\tname: \"res3b1_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b1_branch2c\"\n\ttop: \"res3b1_branch2c\"\n\tname: \"bn3b1_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b1_branch2c\"\n\ttop: \"res3b1_branch2c\"\n\tname: \"scale3b1_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a\"\n\tbottom: \"res3b1_branch2c\"\n\ttop: \"res3b1\"\n\tname: \"res3b1\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3b1\"\n\ttop: \"res3b1\"\n\tname: \"res3b1_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b1\"\n\ttop: \"res3b2_branch2a\"\n\tname: \"res3b2_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b2_branch2a\"\n\ttop: \"res3b2_branch2a\"\n\tname: \"bn3b2_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b2_branch2a\"\n\ttop: \"res3b2_branch2a\"\n\tname: \"scale3b2_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b2_branch2a\"\n\tbottom: \"res3b2_branch2a\"\n\tname: \"res3b2_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b2_branch2a\"\n\ttop: \"res3b2_branch2b\"\n\tname: \"res3b2_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b2_branch2b\"\n\ttop: \"res3b2_branch2b\"\n\tname: \"bn3b2_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b2_branch2b\"\n\ttop: \"res3b2_branch2b\"\n\tname: \"scale3b2_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b2_branch2b\"\n\tbottom: \"res3b2_branch2b\"\n\tname: \"res3b2_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b2_branch2b\"\n\ttop: \"res3b2_branch2c\"\n\tname: \"res3b2_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b2_branch2c\"\n\ttop: \"res3b2_branch2c\"\n\tname: \"bn3b2_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b2_branch2c\"\n\ttop: \"res3b2_branch2c\"\n\tname: \"scale3b2_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b1\"\n\tbottom: \"res3b2_branch2c\"\n\ttop: \"res3b2\"\n\tname: \"res3b2\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3b2\"\n\ttop: \"res3b2\"\n\tname: \"res3b2_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b2\"\n\ttop: \"res3b3_branch2a\"\n\tname: \"res3b3_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b3_branch2a\"\n\ttop: \"res3b3_branch2a\"\n\tname: \"bn3b3_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b3_branch2a\"\n\ttop: \"res3b3_branch2a\"\n\tname: \"scale3b3_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b3_branch2a\"\n\tbottom: \"res3b3_branch2a\"\n\tname: \"res3b3_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b3_branch2a\"\n\ttop: \"res3b3_branch2b\"\n\tname: \"res3b3_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b3_branch2b\"\n\ttop: \"res3b3_branch2b\"\n\tname: \"bn3b3_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b3_branch2b\"\n\ttop: \"res3b3_branch2b\"\n\tname: \"scale3b3_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b3_branch2b\"\n\tbottom: \"res3b3_branch2b\"\n\tname: \"res3b3_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b3_branch2b\"\n\ttop: \"res3b3_branch2c\"\n\tname: \"res3b3_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b3_branch2c\"\n\ttop: \"res3b3_branch2c\"\n\tname: \"bn3b3_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b3_branch2c\"\n\ttop: \"res3b3_branch2c\"\n\tname: \"scale3b3_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b2\"\n\tbottom: \"res3b3_branch2c\"\n\ttop: \"res3b3\"\n\tname: \"res3b3\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3b3\"\n\ttop: \"res3b3\"\n\tname: \"res3b3_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b3\"\n\ttop: \"res3b4_branch2a\"\n\tname: \"res3b4_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b4_branch2a\"\n\ttop: \"res3b4_branch2a\"\n\tname: \"bn3b4_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b4_branch2a\"\n\ttop: \"res3b4_branch2a\"\n\tname: \"scale3b4_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b4_branch2a\"\n\tbottom: \"res3b4_branch2a\"\n\tname: \"res3b4_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b4_branch2a\"\n\ttop: \"res3b4_branch2b\"\n\tname: \"res3b4_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b4_branch2b\"\n\ttop: \"res3b4_branch2b\"\n\tname: \"bn3b4_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b4_branch2b\"\n\ttop: \"res3b4_branch2b\"\n\tname: \"scale3b4_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b4_branch2b\"\n\tbottom: \"res3b4_branch2b\"\n\tname: \"res3b4_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b4_branch2b\"\n\ttop: \"res3b4_branch2c\"\n\tname: \"res3b4_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b4_branch2c\"\n\ttop: \"res3b4_branch2c\"\n\tname: \"bn3b4_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b4_branch2c\"\n\ttop: \"res3b4_branch2c\"\n\tname: \"scale3b4_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b3\"\n\tbottom: \"res3b4_branch2c\"\n\ttop: \"res3b4\"\n\tname: \"res3b4\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3b4\"\n\ttop: \"res3b4\"\n\tname: \"res3b4_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b4\"\n\ttop: \"res3b5_branch2a\"\n\tname: \"res3b5_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b5_branch2a\"\n\ttop: \"res3b5_branch2a\"\n\tname: \"bn3b5_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b5_branch2a\"\n\ttop: \"res3b5_branch2a\"\n\tname: \"scale3b5_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b5_branch2a\"\n\tbottom: \"res3b5_branch2a\"\n\tname: \"res3b5_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b5_branch2a\"\n\ttop: \"res3b5_branch2b\"\n\tname: \"res3b5_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b5_branch2b\"\n\ttop: \"res3b5_branch2b\"\n\tname: \"bn3b5_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b5_branch2b\"\n\ttop: \"res3b5_branch2b\"\n\tname: \"scale3b5_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b5_branch2b\"\n\tbottom: \"res3b5_branch2b\"\n\tname: \"res3b5_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b5_branch2b\"\n\ttop: \"res3b5_branch2c\"\n\tname: \"res3b5_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b5_branch2c\"\n\ttop: \"res3b5_branch2c\"\n\tname: \"bn3b5_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b5_branch2c\"\n\ttop: \"res3b5_branch2c\"\n\tname: \"scale3b5_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b4\"\n\tbottom: \"res3b5_branch2c\"\n\ttop: \"res3b5\"\n\tname: \"res3b5\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3b5\"\n\ttop: \"res3b5\"\n\tname: \"res3b5_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b5\"\n\ttop: \"res3b6_branch2a\"\n\tname: \"res3b6_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b6_branch2a\"\n\ttop: \"res3b6_branch2a\"\n\tname: \"bn3b6_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b6_branch2a\"\n\ttop: \"res3b6_branch2a\"\n\tname: \"scale3b6_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b6_branch2a\"\n\tbottom: \"res3b6_branch2a\"\n\tname: \"res3b6_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b6_branch2a\"\n\ttop: \"res3b6_branch2b\"\n\tname: \"res3b6_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b6_branch2b\"\n\ttop: \"res3b6_branch2b\"\n\tname: \"bn3b6_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b6_branch2b\"\n\ttop: \"res3b6_branch2b\"\n\tname: \"scale3b6_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b6_branch2b\"\n\tbottom: \"res3b6_branch2b\"\n\tname: \"res3b6_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b6_branch2b\"\n\ttop: \"res3b6_branch2c\"\n\tname: \"res3b6_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b6_branch2c\"\n\ttop: \"res3b6_branch2c\"\n\tname: \"bn3b6_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b6_branch2c\"\n\ttop: \"res3b6_branch2c\"\n\tname: \"scale3b6_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b5\"\n\tbottom: \"res3b6_branch2c\"\n\ttop: \"res3b6\"\n\tname: \"res3b6\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3b6\"\n\ttop: \"res3b6\"\n\tname: \"res3b6_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b6\"\n\ttop: \"res3b7_branch2a\"\n\tname: \"res3b7_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b7_branch2a\"\n\ttop: \"res3b7_branch2a\"\n\tname: \"bn3b7_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b7_branch2a\"\n\ttop: \"res3b7_branch2a\"\n\tname: \"scale3b7_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b7_branch2a\"\n\tbottom: \"res3b7_branch2a\"\n\tname: \"res3b7_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b7_branch2a\"\n\ttop: \"res3b7_branch2b\"\n\tname: \"res3b7_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b7_branch2b\"\n\ttop: \"res3b7_branch2b\"\n\tname: \"bn3b7_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b7_branch2b\"\n\ttop: \"res3b7_branch2b\"\n\tname: \"scale3b7_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res3b7_branch2b\"\n\tbottom: \"res3b7_branch2b\"\n\tname: \"res3b7_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b7_branch2b\"\n\ttop: \"res3b7_branch2c\"\n\tname: \"res3b7_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b7_branch2c\"\n\ttop: \"res3b7_branch2c\"\n\tname: \"bn3b7_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b7_branch2c\"\n\ttop: \"res3b7_branch2c\"\n\tname: \"scale3b7_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b6\"\n\tbottom: \"res3b7_branch2c\"\n\ttop: \"res3b7\"\n\tname: \"res3b7\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3b7\"\n\ttop: \"res3b7\"\n\tname: \"res3b7_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b7\"\n\ttop: \"res4a_branch1\"\n\tname: \"res4a_branch1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch1\"\n\ttop: \"res4a_branch1\"\n\tname: \"bn4a_branch1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch1\"\n\ttop: \"res4a_branch1\"\n\tname: \"scale4a_branch1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b7\"\n\ttop: \"res4a_branch2a\"\n\tname: \"res4a_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2a\"\n\ttop: \"res4a_branch2a\"\n\tname: \"bn4a_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2a\"\n\ttop: \"res4a_branch2a\"\n\tname: \"scale4a_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4a_branch2a\"\n\tbottom: \"res4a_branch2a\"\n\tname: \"res4a_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4a_branch2a\"\n\ttop: \"res4a_branch2b\"\n\tname: \"res4a_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2b\"\n\ttop: \"res4a_branch2b\"\n\tname: \"bn4a_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2b\"\n\ttop: \"res4a_branch2b\"\n\tname: \"scale4a_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4a_branch2b\"\n\tbottom: \"res4a_branch2b\"\n\tname: \"res4a_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4a_branch2b\"\n\ttop: \"res4a_branch2c\"\n\tname: \"res4a_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2c\"\n\ttop: \"res4a_branch2c\"\n\tname: \"bn4a_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2c\"\n\ttop: \"res4a_branch2c\"\n\tname: \"scale4a_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch1\"\n\tbottom: \"res4a_branch2c\"\n\ttop: \"res4a\"\n\tname: \"res4a\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4a\"\n\ttop: \"res4a\"\n\tname: \"res4a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4a\"\n\ttop: \"res4b1_branch2a\"\n\tname: \"res4b1_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b1_branch2a\"\n\ttop: \"res4b1_branch2a\"\n\tname: \"bn4b1_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b1_branch2a\"\n\ttop: \"res4b1_branch2a\"\n\tname: \"scale4b1_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b1_branch2a\"\n\tbottom: \"res4b1_branch2a\"\n\tname: \"res4b1_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b1_branch2a\"\n\ttop: \"res4b1_branch2b\"\n\tname: \"res4b1_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b1_branch2b\"\n\ttop: \"res4b1_branch2b\"\n\tname: \"bn4b1_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b1_branch2b\"\n\ttop: \"res4b1_branch2b\"\n\tname: \"scale4b1_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b1_branch2b\"\n\tbottom: \"res4b1_branch2b\"\n\tname: \"res4b1_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b1_branch2b\"\n\ttop: \"res4b1_branch2c\"\n\tname: \"res4b1_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b1_branch2c\"\n\ttop: \"res4b1_branch2c\"\n\tname: \"bn4b1_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b1_branch2c\"\n\ttop: \"res4b1_branch2c\"\n\tname: \"scale4b1_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a\"\n\tbottom: \"res4b1_branch2c\"\n\ttop: \"res4b1\"\n\tname: \"res4b1\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b1\"\n\ttop: \"res4b1\"\n\tname: \"res4b1_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b1\"\n\ttop: \"res4b2_branch2a\"\n\tname: \"res4b2_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b2_branch2a\"\n\ttop: \"res4b2_branch2a\"\n\tname: \"bn4b2_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b2_branch2a\"\n\ttop: \"res4b2_branch2a\"\n\tname: \"scale4b2_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b2_branch2a\"\n\tbottom: \"res4b2_branch2a\"\n\tname: \"res4b2_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b2_branch2a\"\n\ttop: \"res4b2_branch2b\"\n\tname: \"res4b2_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b2_branch2b\"\n\ttop: \"res4b2_branch2b\"\n\tname: \"bn4b2_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b2_branch2b\"\n\ttop: \"res4b2_branch2b\"\n\tname: \"scale4b2_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b2_branch2b\"\n\tbottom: \"res4b2_branch2b\"\n\tname: \"res4b2_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b2_branch2b\"\n\ttop: \"res4b2_branch2c\"\n\tname: \"res4b2_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b2_branch2c\"\n\ttop: \"res4b2_branch2c\"\n\tname: \"bn4b2_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b2_branch2c\"\n\ttop: \"res4b2_branch2c\"\n\tname: \"scale4b2_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b1\"\n\tbottom: \"res4b2_branch2c\"\n\ttop: \"res4b2\"\n\tname: \"res4b2\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b2\"\n\ttop: \"res4b2\"\n\tname: \"res4b2_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b2\"\n\ttop: \"res4b3_branch2a\"\n\tname: \"res4b3_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b3_branch2a\"\n\ttop: \"res4b3_branch2a\"\n\tname: \"bn4b3_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b3_branch2a\"\n\ttop: \"res4b3_branch2a\"\n\tname: \"scale4b3_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b3_branch2a\"\n\tbottom: \"res4b3_branch2a\"\n\tname: \"res4b3_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b3_branch2a\"\n\ttop: \"res4b3_branch2b\"\n\tname: \"res4b3_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b3_branch2b\"\n\ttop: \"res4b3_branch2b\"\n\tname: \"bn4b3_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b3_branch2b\"\n\ttop: \"res4b3_branch2b\"\n\tname: \"scale4b3_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b3_branch2b\"\n\tbottom: \"res4b3_branch2b\"\n\tname: \"res4b3_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b3_branch2b\"\n\ttop: \"res4b3_branch2c\"\n\tname: \"res4b3_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b3_branch2c\"\n\ttop: \"res4b3_branch2c\"\n\tname: \"bn4b3_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b3_branch2c\"\n\ttop: \"res4b3_branch2c\"\n\tname: \"scale4b3_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b2\"\n\tbottom: \"res4b3_branch2c\"\n\ttop: \"res4b3\"\n\tname: \"res4b3\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b3\"\n\ttop: \"res4b3\"\n\tname: \"res4b3_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b3\"\n\ttop: \"res4b4_branch2a\"\n\tname: \"res4b4_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b4_branch2a\"\n\ttop: \"res4b4_branch2a\"\n\tname: \"bn4b4_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b4_branch2a\"\n\ttop: \"res4b4_branch2a\"\n\tname: \"scale4b4_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b4_branch2a\"\n\tbottom: \"res4b4_branch2a\"\n\tname: \"res4b4_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b4_branch2a\"\n\ttop: \"res4b4_branch2b\"\n\tname: \"res4b4_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b4_branch2b\"\n\ttop: \"res4b4_branch2b\"\n\tname: \"bn4b4_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b4_branch2b\"\n\ttop: \"res4b4_branch2b\"\n\tname: \"scale4b4_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b4_branch2b\"\n\tbottom: \"res4b4_branch2b\"\n\tname: \"res4b4_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b4_branch2b\"\n\ttop: \"res4b4_branch2c\"\n\tname: \"res4b4_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b4_branch2c\"\n\ttop: \"res4b4_branch2c\"\n\tname: \"bn4b4_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b4_branch2c\"\n\ttop: \"res4b4_branch2c\"\n\tname: \"scale4b4_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b3\"\n\tbottom: \"res4b4_branch2c\"\n\ttop: \"res4b4\"\n\tname: \"res4b4\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b4\"\n\ttop: \"res4b4\"\n\tname: \"res4b4_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b4\"\n\ttop: \"res4b5_branch2a\"\n\tname: \"res4b5_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b5_branch2a\"\n\ttop: \"res4b5_branch2a\"\n\tname: \"bn4b5_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b5_branch2a\"\n\ttop: \"res4b5_branch2a\"\n\tname: \"scale4b5_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b5_branch2a\"\n\tbottom: \"res4b5_branch2a\"\n\tname: \"res4b5_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b5_branch2a\"\n\ttop: \"res4b5_branch2b\"\n\tname: \"res4b5_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b5_branch2b\"\n\ttop: \"res4b5_branch2b\"\n\tname: \"bn4b5_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b5_branch2b\"\n\ttop: \"res4b5_branch2b\"\n\tname: \"scale4b5_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b5_branch2b\"\n\tbottom: \"res4b5_branch2b\"\n\tname: \"res4b5_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b5_branch2b\"\n\ttop: \"res4b5_branch2c\"\n\tname: \"res4b5_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b5_branch2c\"\n\ttop: \"res4b5_branch2c\"\n\tname: \"bn4b5_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b5_branch2c\"\n\ttop: \"res4b5_branch2c\"\n\tname: \"scale4b5_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b4\"\n\tbottom: \"res4b5_branch2c\"\n\ttop: \"res4b5\"\n\tname: \"res4b5\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b5\"\n\ttop: \"res4b5\"\n\tname: \"res4b5_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b5\"\n\ttop: \"res4b6_branch2a\"\n\tname: \"res4b6_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b6_branch2a\"\n\ttop: \"res4b6_branch2a\"\n\tname: \"bn4b6_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b6_branch2a\"\n\ttop: \"res4b6_branch2a\"\n\tname: \"scale4b6_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b6_branch2a\"\n\tbottom: \"res4b6_branch2a\"\n\tname: \"res4b6_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b6_branch2a\"\n\ttop: \"res4b6_branch2b\"\n\tname: \"res4b6_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b6_branch2b\"\n\ttop: \"res4b6_branch2b\"\n\tname: \"bn4b6_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b6_branch2b\"\n\ttop: \"res4b6_branch2b\"\n\tname: \"scale4b6_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b6_branch2b\"\n\tbottom: \"res4b6_branch2b\"\n\tname: \"res4b6_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b6_branch2b\"\n\ttop: \"res4b6_branch2c\"\n\tname: \"res4b6_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b6_branch2c\"\n\ttop: \"res4b6_branch2c\"\n\tname: \"bn4b6_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b6_branch2c\"\n\ttop: \"res4b6_branch2c\"\n\tname: \"scale4b6_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b5\"\n\tbottom: \"res4b6_branch2c\"\n\ttop: \"res4b6\"\n\tname: \"res4b6\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b6\"\n\ttop: \"res4b6\"\n\tname: \"res4b6_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b6\"\n\ttop: \"res4b7_branch2a\"\n\tname: \"res4b7_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b7_branch2a\"\n\ttop: \"res4b7_branch2a\"\n\tname: \"bn4b7_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b7_branch2a\"\n\ttop: \"res4b7_branch2a\"\n\tname: \"scale4b7_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b7_branch2a\"\n\tbottom: \"res4b7_branch2a\"\n\tname: \"res4b7_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b7_branch2a\"\n\ttop: \"res4b7_branch2b\"\n\tname: \"res4b7_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b7_branch2b\"\n\ttop: \"res4b7_branch2b\"\n\tname: \"bn4b7_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b7_branch2b\"\n\ttop: \"res4b7_branch2b\"\n\tname: \"scale4b7_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b7_branch2b\"\n\tbottom: \"res4b7_branch2b\"\n\tname: \"res4b7_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b7_branch2b\"\n\ttop: \"res4b7_branch2c\"\n\tname: \"res4b7_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b7_branch2c\"\n\ttop: \"res4b7_branch2c\"\n\tname: \"bn4b7_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b7_branch2c\"\n\ttop: \"res4b7_branch2c\"\n\tname: \"scale4b7_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b6\"\n\tbottom: \"res4b7_branch2c\"\n\ttop: \"res4b7\"\n\tname: \"res4b7\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b7\"\n\ttop: \"res4b7\"\n\tname: \"res4b7_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b7\"\n\ttop: \"res4b8_branch2a\"\n\tname: \"res4b8_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b8_branch2a\"\n\ttop: \"res4b8_branch2a\"\n\tname: \"bn4b8_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b8_branch2a\"\n\ttop: \"res4b8_branch2a\"\n\tname: \"scale4b8_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b8_branch2a\"\n\tbottom: \"res4b8_branch2a\"\n\tname: \"res4b8_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b8_branch2a\"\n\ttop: \"res4b8_branch2b\"\n\tname: \"res4b8_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b8_branch2b\"\n\ttop: \"res4b8_branch2b\"\n\tname: \"bn4b8_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b8_branch2b\"\n\ttop: \"res4b8_branch2b\"\n\tname: \"scale4b8_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b8_branch2b\"\n\tbottom: \"res4b8_branch2b\"\n\tname: \"res4b8_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b8_branch2b\"\n\ttop: \"res4b8_branch2c\"\n\tname: \"res4b8_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b8_branch2c\"\n\ttop: \"res4b8_branch2c\"\n\tname: \"bn4b8_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b8_branch2c\"\n\ttop: \"res4b8_branch2c\"\n\tname: \"scale4b8_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b7\"\n\tbottom: \"res4b8_branch2c\"\n\ttop: \"res4b8\"\n\tname: \"res4b8\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b8\"\n\ttop: \"res4b8\"\n\tname: \"res4b8_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b8\"\n\ttop: \"res4b9_branch2a\"\n\tname: \"res4b9_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b9_branch2a\"\n\ttop: \"res4b9_branch2a\"\n\tname: \"bn4b9_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b9_branch2a\"\n\ttop: \"res4b9_branch2a\"\n\tname: \"scale4b9_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b9_branch2a\"\n\tbottom: \"res4b9_branch2a\"\n\tname: \"res4b9_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b9_branch2a\"\n\ttop: \"res4b9_branch2b\"\n\tname: \"res4b9_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b9_branch2b\"\n\ttop: \"res4b9_branch2b\"\n\tname: \"bn4b9_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b9_branch2b\"\n\ttop: \"res4b9_branch2b\"\n\tname: \"scale4b9_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b9_branch2b\"\n\tbottom: \"res4b9_branch2b\"\n\tname: \"res4b9_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b9_branch2b\"\n\ttop: \"res4b9_branch2c\"\n\tname: \"res4b9_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b9_branch2c\"\n\ttop: \"res4b9_branch2c\"\n\tname: \"bn4b9_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b9_branch2c\"\n\ttop: \"res4b9_branch2c\"\n\tname: \"scale4b9_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b8\"\n\tbottom: \"res4b9_branch2c\"\n\ttop: \"res4b9\"\n\tname: \"res4b9\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b9\"\n\ttop: \"res4b9\"\n\tname: \"res4b9_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b9\"\n\ttop: \"res4b10_branch2a\"\n\tname: \"res4b10_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b10_branch2a\"\n\ttop: \"res4b10_branch2a\"\n\tname: \"bn4b10_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b10_branch2a\"\n\ttop: \"res4b10_branch2a\"\n\tname: \"scale4b10_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b10_branch2a\"\n\tbottom: \"res4b10_branch2a\"\n\tname: \"res4b10_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b10_branch2a\"\n\ttop: \"res4b10_branch2b\"\n\tname: \"res4b10_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b10_branch2b\"\n\ttop: \"res4b10_branch2b\"\n\tname: \"bn4b10_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b10_branch2b\"\n\ttop: \"res4b10_branch2b\"\n\tname: \"scale4b10_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b10_branch2b\"\n\tbottom: \"res4b10_branch2b\"\n\tname: \"res4b10_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b10_branch2b\"\n\ttop: \"res4b10_branch2c\"\n\tname: \"res4b10_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b10_branch2c\"\n\ttop: \"res4b10_branch2c\"\n\tname: \"bn4b10_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b10_branch2c\"\n\ttop: \"res4b10_branch2c\"\n\tname: \"scale4b10_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b9\"\n\tbottom: \"res4b10_branch2c\"\n\ttop: \"res4b10\"\n\tname: \"res4b10\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b10\"\n\ttop: \"res4b10\"\n\tname: \"res4b10_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b10\"\n\ttop: \"res4b11_branch2a\"\n\tname: \"res4b11_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b11_branch2a\"\n\ttop: \"res4b11_branch2a\"\n\tname: \"bn4b11_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b11_branch2a\"\n\ttop: \"res4b11_branch2a\"\n\tname: \"scale4b11_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b11_branch2a\"\n\tbottom: \"res4b11_branch2a\"\n\tname: \"res4b11_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b11_branch2a\"\n\ttop: \"res4b11_branch2b\"\n\tname: \"res4b11_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b11_branch2b\"\n\ttop: \"res4b11_branch2b\"\n\tname: \"bn4b11_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b11_branch2b\"\n\ttop: \"res4b11_branch2b\"\n\tname: \"scale4b11_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b11_branch2b\"\n\tbottom: \"res4b11_branch2b\"\n\tname: \"res4b11_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b11_branch2b\"\n\ttop: \"res4b11_branch2c\"\n\tname: \"res4b11_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b11_branch2c\"\n\ttop: \"res4b11_branch2c\"\n\tname: \"bn4b11_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b11_branch2c\"\n\ttop: \"res4b11_branch2c\"\n\tname: \"scale4b11_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b10\"\n\tbottom: \"res4b11_branch2c\"\n\ttop: \"res4b11\"\n\tname: \"res4b11\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b11\"\n\ttop: \"res4b11\"\n\tname: \"res4b11_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b11\"\n\ttop: \"res4b12_branch2a\"\n\tname: \"res4b12_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b12_branch2a\"\n\ttop: \"res4b12_branch2a\"\n\tname: \"bn4b12_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b12_branch2a\"\n\ttop: \"res4b12_branch2a\"\n\tname: \"scale4b12_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b12_branch2a\"\n\tbottom: \"res4b12_branch2a\"\n\tname: \"res4b12_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b12_branch2a\"\n\ttop: \"res4b12_branch2b\"\n\tname: \"res4b12_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b12_branch2b\"\n\ttop: \"res4b12_branch2b\"\n\tname: \"bn4b12_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b12_branch2b\"\n\ttop: \"res4b12_branch2b\"\n\tname: \"scale4b12_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b12_branch2b\"\n\tbottom: \"res4b12_branch2b\"\n\tname: \"res4b12_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b12_branch2b\"\n\ttop: \"res4b12_branch2c\"\n\tname: \"res4b12_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b12_branch2c\"\n\ttop: \"res4b12_branch2c\"\n\tname: \"bn4b12_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b12_branch2c\"\n\ttop: \"res4b12_branch2c\"\n\tname: \"scale4b12_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b11\"\n\tbottom: \"res4b12_branch2c\"\n\ttop: \"res4b12\"\n\tname: \"res4b12\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b12\"\n\ttop: \"res4b12\"\n\tname: \"res4b12_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b12\"\n\ttop: \"res4b13_branch2a\"\n\tname: \"res4b13_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b13_branch2a\"\n\ttop: \"res4b13_branch2a\"\n\tname: \"bn4b13_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b13_branch2a\"\n\ttop: \"res4b13_branch2a\"\n\tname: \"scale4b13_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b13_branch2a\"\n\tbottom: \"res4b13_branch2a\"\n\tname: \"res4b13_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b13_branch2a\"\n\ttop: \"res4b13_branch2b\"\n\tname: \"res4b13_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b13_branch2b\"\n\ttop: \"res4b13_branch2b\"\n\tname: \"bn4b13_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b13_branch2b\"\n\ttop: \"res4b13_branch2b\"\n\tname: \"scale4b13_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b13_branch2b\"\n\tbottom: \"res4b13_branch2b\"\n\tname: \"res4b13_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b13_branch2b\"\n\ttop: \"res4b13_branch2c\"\n\tname: \"res4b13_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b13_branch2c\"\n\ttop: \"res4b13_branch2c\"\n\tname: \"bn4b13_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b13_branch2c\"\n\ttop: \"res4b13_branch2c\"\n\tname: \"scale4b13_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b12\"\n\tbottom: \"res4b13_branch2c\"\n\ttop: \"res4b13\"\n\tname: \"res4b13\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b13\"\n\ttop: \"res4b13\"\n\tname: \"res4b13_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b13\"\n\ttop: \"res4b14_branch2a\"\n\tname: \"res4b14_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b14_branch2a\"\n\ttop: \"res4b14_branch2a\"\n\tname: \"bn4b14_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b14_branch2a\"\n\ttop: \"res4b14_branch2a\"\n\tname: \"scale4b14_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b14_branch2a\"\n\tbottom: \"res4b14_branch2a\"\n\tname: \"res4b14_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b14_branch2a\"\n\ttop: \"res4b14_branch2b\"\n\tname: \"res4b14_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b14_branch2b\"\n\ttop: \"res4b14_branch2b\"\n\tname: \"bn4b14_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b14_branch2b\"\n\ttop: \"res4b14_branch2b\"\n\tname: \"scale4b14_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b14_branch2b\"\n\tbottom: \"res4b14_branch2b\"\n\tname: \"res4b14_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b14_branch2b\"\n\ttop: \"res4b14_branch2c\"\n\tname: \"res4b14_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b14_branch2c\"\n\ttop: \"res4b14_branch2c\"\n\tname: \"bn4b14_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b14_branch2c\"\n\ttop: \"res4b14_branch2c\"\n\tname: \"scale4b14_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b13\"\n\tbottom: \"res4b14_branch2c\"\n\ttop: \"res4b14\"\n\tname: \"res4b14\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b14\"\n\ttop: \"res4b14\"\n\tname: \"res4b14_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b14\"\n\ttop: \"res4b15_branch2a\"\n\tname: \"res4b15_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b15_branch2a\"\n\ttop: \"res4b15_branch2a\"\n\tname: \"bn4b15_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b15_branch2a\"\n\ttop: \"res4b15_branch2a\"\n\tname: \"scale4b15_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b15_branch2a\"\n\tbottom: \"res4b15_branch2a\"\n\tname: \"res4b15_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b15_branch2a\"\n\ttop: \"res4b15_branch2b\"\n\tname: \"res4b15_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b15_branch2b\"\n\ttop: \"res4b15_branch2b\"\n\tname: \"bn4b15_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b15_branch2b\"\n\ttop: \"res4b15_branch2b\"\n\tname: \"scale4b15_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b15_branch2b\"\n\tbottom: \"res4b15_branch2b\"\n\tname: \"res4b15_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b15_branch2b\"\n\ttop: \"res4b15_branch2c\"\n\tname: \"res4b15_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b15_branch2c\"\n\ttop: \"res4b15_branch2c\"\n\tname: \"bn4b15_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b15_branch2c\"\n\ttop: \"res4b15_branch2c\"\n\tname: \"scale4b15_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b14\"\n\tbottom: \"res4b15_branch2c\"\n\ttop: \"res4b15\"\n\tname: \"res4b15\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b15\"\n\ttop: \"res4b15\"\n\tname: \"res4b15_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b15\"\n\ttop: \"res4b16_branch2a\"\n\tname: \"res4b16_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b16_branch2a\"\n\ttop: \"res4b16_branch2a\"\n\tname: \"bn4b16_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b16_branch2a\"\n\ttop: \"res4b16_branch2a\"\n\tname: \"scale4b16_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b16_branch2a\"\n\tbottom: \"res4b16_branch2a\"\n\tname: \"res4b16_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b16_branch2a\"\n\ttop: \"res4b16_branch2b\"\n\tname: \"res4b16_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b16_branch2b\"\n\ttop: \"res4b16_branch2b\"\n\tname: \"bn4b16_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b16_branch2b\"\n\ttop: \"res4b16_branch2b\"\n\tname: \"scale4b16_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b16_branch2b\"\n\tbottom: \"res4b16_branch2b\"\n\tname: \"res4b16_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b16_branch2b\"\n\ttop: \"res4b16_branch2c\"\n\tname: \"res4b16_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b16_branch2c\"\n\ttop: \"res4b16_branch2c\"\n\tname: \"bn4b16_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b16_branch2c\"\n\ttop: \"res4b16_branch2c\"\n\tname: \"scale4b16_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b15\"\n\tbottom: \"res4b16_branch2c\"\n\ttop: \"res4b16\"\n\tname: \"res4b16\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b16\"\n\ttop: \"res4b16\"\n\tname: \"res4b16_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b16\"\n\ttop: \"res4b17_branch2a\"\n\tname: \"res4b17_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b17_branch2a\"\n\ttop: \"res4b17_branch2a\"\n\tname: \"bn4b17_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b17_branch2a\"\n\ttop: \"res4b17_branch2a\"\n\tname: \"scale4b17_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b17_branch2a\"\n\tbottom: \"res4b17_branch2a\"\n\tname: \"res4b17_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b17_branch2a\"\n\ttop: \"res4b17_branch2b\"\n\tname: \"res4b17_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b17_branch2b\"\n\ttop: \"res4b17_branch2b\"\n\tname: \"bn4b17_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b17_branch2b\"\n\ttop: \"res4b17_branch2b\"\n\tname: \"scale4b17_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b17_branch2b\"\n\tbottom: \"res4b17_branch2b\"\n\tname: \"res4b17_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b17_branch2b\"\n\ttop: \"res4b17_branch2c\"\n\tname: \"res4b17_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b17_branch2c\"\n\ttop: \"res4b17_branch2c\"\n\tname: \"bn4b17_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b17_branch2c\"\n\ttop: \"res4b17_branch2c\"\n\tname: \"scale4b17_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b16\"\n\tbottom: \"res4b17_branch2c\"\n\ttop: \"res4b17\"\n\tname: \"res4b17\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b17\"\n\ttop: \"res4b17\"\n\tname: \"res4b17_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b17\"\n\ttop: \"res4b18_branch2a\"\n\tname: \"res4b18_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b18_branch2a\"\n\ttop: \"res4b18_branch2a\"\n\tname: \"bn4b18_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b18_branch2a\"\n\ttop: \"res4b18_branch2a\"\n\tname: \"scale4b18_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b18_branch2a\"\n\tbottom: \"res4b18_branch2a\"\n\tname: \"res4b18_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b18_branch2a\"\n\ttop: \"res4b18_branch2b\"\n\tname: \"res4b18_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b18_branch2b\"\n\ttop: \"res4b18_branch2b\"\n\tname: \"bn4b18_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b18_branch2b\"\n\ttop: \"res4b18_branch2b\"\n\tname: \"scale4b18_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b18_branch2b\"\n\tbottom: \"res4b18_branch2b\"\n\tname: \"res4b18_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b18_branch2b\"\n\ttop: \"res4b18_branch2c\"\n\tname: \"res4b18_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b18_branch2c\"\n\ttop: \"res4b18_branch2c\"\n\tname: \"bn4b18_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b18_branch2c\"\n\ttop: \"res4b18_branch2c\"\n\tname: \"scale4b18_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b17\"\n\tbottom: \"res4b18_branch2c\"\n\ttop: \"res4b18\"\n\tname: \"res4b18\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b18\"\n\ttop: \"res4b18\"\n\tname: \"res4b18_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b18\"\n\ttop: \"res4b19_branch2a\"\n\tname: \"res4b19_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b19_branch2a\"\n\ttop: \"res4b19_branch2a\"\n\tname: \"bn4b19_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b19_branch2a\"\n\ttop: \"res4b19_branch2a\"\n\tname: \"scale4b19_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b19_branch2a\"\n\tbottom: \"res4b19_branch2a\"\n\tname: \"res4b19_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b19_branch2a\"\n\ttop: \"res4b19_branch2b\"\n\tname: \"res4b19_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b19_branch2b\"\n\ttop: \"res4b19_branch2b\"\n\tname: \"bn4b19_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b19_branch2b\"\n\ttop: \"res4b19_branch2b\"\n\tname: \"scale4b19_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b19_branch2b\"\n\tbottom: \"res4b19_branch2b\"\n\tname: \"res4b19_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b19_branch2b\"\n\ttop: \"res4b19_branch2c\"\n\tname: \"res4b19_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b19_branch2c\"\n\ttop: \"res4b19_branch2c\"\n\tname: \"bn4b19_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b19_branch2c\"\n\ttop: \"res4b19_branch2c\"\n\tname: \"scale4b19_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b18\"\n\tbottom: \"res4b19_branch2c\"\n\ttop: \"res4b19\"\n\tname: \"res4b19\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b19\"\n\ttop: \"res4b19\"\n\tname: \"res4b19_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b19\"\n\ttop: \"res4b20_branch2a\"\n\tname: \"res4b20_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b20_branch2a\"\n\ttop: \"res4b20_branch2a\"\n\tname: \"bn4b20_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b20_branch2a\"\n\ttop: \"res4b20_branch2a\"\n\tname: \"scale4b20_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b20_branch2a\"\n\tbottom: \"res4b20_branch2a\"\n\tname: \"res4b20_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b20_branch2a\"\n\ttop: \"res4b20_branch2b\"\n\tname: \"res4b20_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b20_branch2b\"\n\ttop: \"res4b20_branch2b\"\n\tname: \"bn4b20_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b20_branch2b\"\n\ttop: \"res4b20_branch2b\"\n\tname: \"scale4b20_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b20_branch2b\"\n\tbottom: \"res4b20_branch2b\"\n\tname: \"res4b20_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b20_branch2b\"\n\ttop: \"res4b20_branch2c\"\n\tname: \"res4b20_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b20_branch2c\"\n\ttop: \"res4b20_branch2c\"\n\tname: \"bn4b20_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b20_branch2c\"\n\ttop: \"res4b20_branch2c\"\n\tname: \"scale4b20_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b19\"\n\tbottom: \"res4b20_branch2c\"\n\ttop: \"res4b20\"\n\tname: \"res4b20\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b20\"\n\ttop: \"res4b20\"\n\tname: \"res4b20_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b20\"\n\ttop: \"res4b21_branch2a\"\n\tname: \"res4b21_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b21_branch2a\"\n\ttop: \"res4b21_branch2a\"\n\tname: \"bn4b21_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b21_branch2a\"\n\ttop: \"res4b21_branch2a\"\n\tname: \"scale4b21_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b21_branch2a\"\n\tbottom: \"res4b21_branch2a\"\n\tname: \"res4b21_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b21_branch2a\"\n\ttop: \"res4b21_branch2b\"\n\tname: \"res4b21_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b21_branch2b\"\n\ttop: \"res4b21_branch2b\"\n\tname: \"bn4b21_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b21_branch2b\"\n\ttop: \"res4b21_branch2b\"\n\tname: \"scale4b21_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b21_branch2b\"\n\tbottom: \"res4b21_branch2b\"\n\tname: \"res4b21_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b21_branch2b\"\n\ttop: \"res4b21_branch2c\"\n\tname: \"res4b21_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b21_branch2c\"\n\ttop: \"res4b21_branch2c\"\n\tname: \"bn4b21_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b21_branch2c\"\n\ttop: \"res4b21_branch2c\"\n\tname: \"scale4b21_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b20\"\n\tbottom: \"res4b21_branch2c\"\n\ttop: \"res4b21\"\n\tname: \"res4b21\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b21\"\n\ttop: \"res4b21\"\n\tname: \"res4b21_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b21\"\n\ttop: \"res4b22_branch2a\"\n\tname: \"res4b22_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b22_branch2a\"\n\ttop: \"res4b22_branch2a\"\n\tname: \"bn4b22_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b22_branch2a\"\n\ttop: \"res4b22_branch2a\"\n\tname: \"scale4b22_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b22_branch2a\"\n\tbottom: \"res4b22_branch2a\"\n\tname: \"res4b22_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b22_branch2a\"\n\ttop: \"res4b22_branch2b\"\n\tname: \"res4b22_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b22_branch2b\"\n\ttop: \"res4b22_branch2b\"\n\tname: \"bn4b22_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b22_branch2b\"\n\ttop: \"res4b22_branch2b\"\n\tname: \"scale4b22_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b22_branch2b\"\n\tbottom: \"res4b22_branch2b\"\n\tname: \"res4b22_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b22_branch2b\"\n\ttop: \"res4b22_branch2c\"\n\tname: \"res4b22_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b22_branch2c\"\n\ttop: \"res4b22_branch2c\"\n\tname: \"bn4b22_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b22_branch2c\"\n\ttop: \"res4b22_branch2c\"\n\tname: \"scale4b22_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b21\"\n\tbottom: \"res4b22_branch2c\"\n\ttop: \"res4b22\"\n\tname: \"res4b22\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b22\"\n\ttop: \"res4b22\"\n\tname: \"res4b22_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b22\"\n\ttop: \"res4b23_branch2a\"\n\tname: \"res4b23_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b23_branch2a\"\n\ttop: \"res4b23_branch2a\"\n\tname: \"bn4b23_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b23_branch2a\"\n\ttop: \"res4b23_branch2a\"\n\tname: \"scale4b23_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b23_branch2a\"\n\tbottom: \"res4b23_branch2a\"\n\tname: \"res4b23_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b23_branch2a\"\n\ttop: \"res4b23_branch2b\"\n\tname: \"res4b23_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b23_branch2b\"\n\ttop: \"res4b23_branch2b\"\n\tname: \"bn4b23_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b23_branch2b\"\n\ttop: \"res4b23_branch2b\"\n\tname: \"scale4b23_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b23_branch2b\"\n\tbottom: \"res4b23_branch2b\"\n\tname: \"res4b23_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b23_branch2b\"\n\ttop: \"res4b23_branch2c\"\n\tname: \"res4b23_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b23_branch2c\"\n\ttop: \"res4b23_branch2c\"\n\tname: \"bn4b23_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b23_branch2c\"\n\ttop: \"res4b23_branch2c\"\n\tname: \"scale4b23_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b22\"\n\tbottom: \"res4b23_branch2c\"\n\ttop: \"res4b23\"\n\tname: \"res4b23\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b23\"\n\ttop: \"res4b23\"\n\tname: \"res4b23_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b23\"\n\ttop: \"res4b24_branch2a\"\n\tname: \"res4b24_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b24_branch2a\"\n\ttop: \"res4b24_branch2a\"\n\tname: \"bn4b24_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b24_branch2a\"\n\ttop: \"res4b24_branch2a\"\n\tname: \"scale4b24_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b24_branch2a\"\n\tbottom: \"res4b24_branch2a\"\n\tname: \"res4b24_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b24_branch2a\"\n\ttop: \"res4b24_branch2b\"\n\tname: \"res4b24_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b24_branch2b\"\n\ttop: \"res4b24_branch2b\"\n\tname: \"bn4b24_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b24_branch2b\"\n\ttop: \"res4b24_branch2b\"\n\tname: \"scale4b24_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b24_branch2b\"\n\tbottom: \"res4b24_branch2b\"\n\tname: \"res4b24_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b24_branch2b\"\n\ttop: \"res4b24_branch2c\"\n\tname: \"res4b24_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b24_branch2c\"\n\ttop: \"res4b24_branch2c\"\n\tname: \"bn4b24_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b24_branch2c\"\n\ttop: \"res4b24_branch2c\"\n\tname: \"scale4b24_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b23\"\n\tbottom: \"res4b24_branch2c\"\n\ttop: \"res4b24\"\n\tname: \"res4b24\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b24\"\n\ttop: \"res4b24\"\n\tname: \"res4b24_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b24\"\n\ttop: \"res4b25_branch2a\"\n\tname: \"res4b25_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b25_branch2a\"\n\ttop: \"res4b25_branch2a\"\n\tname: \"bn4b25_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b25_branch2a\"\n\ttop: \"res4b25_branch2a\"\n\tname: \"scale4b25_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b25_branch2a\"\n\tbottom: \"res4b25_branch2a\"\n\tname: \"res4b25_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b25_branch2a\"\n\ttop: \"res4b25_branch2b\"\n\tname: \"res4b25_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b25_branch2b\"\n\ttop: \"res4b25_branch2b\"\n\tname: \"bn4b25_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b25_branch2b\"\n\ttop: \"res4b25_branch2b\"\n\tname: \"scale4b25_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b25_branch2b\"\n\tbottom: \"res4b25_branch2b\"\n\tname: \"res4b25_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b25_branch2b\"\n\ttop: \"res4b25_branch2c\"\n\tname: \"res4b25_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b25_branch2c\"\n\ttop: \"res4b25_branch2c\"\n\tname: \"bn4b25_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b25_branch2c\"\n\ttop: \"res4b25_branch2c\"\n\tname: \"scale4b25_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b24\"\n\tbottom: \"res4b25_branch2c\"\n\ttop: \"res4b25\"\n\tname: \"res4b25\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b25\"\n\ttop: \"res4b25\"\n\tname: \"res4b25_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b25\"\n\ttop: \"res4b26_branch2a\"\n\tname: \"res4b26_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b26_branch2a\"\n\ttop: \"res4b26_branch2a\"\n\tname: \"bn4b26_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b26_branch2a\"\n\ttop: \"res4b26_branch2a\"\n\tname: \"scale4b26_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b26_branch2a\"\n\tbottom: \"res4b26_branch2a\"\n\tname: \"res4b26_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b26_branch2a\"\n\ttop: \"res4b26_branch2b\"\n\tname: \"res4b26_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b26_branch2b\"\n\ttop: \"res4b26_branch2b\"\n\tname: \"bn4b26_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b26_branch2b\"\n\ttop: \"res4b26_branch2b\"\n\tname: \"scale4b26_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b26_branch2b\"\n\tbottom: \"res4b26_branch2b\"\n\tname: \"res4b26_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b26_branch2b\"\n\ttop: \"res4b26_branch2c\"\n\tname: \"res4b26_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b26_branch2c\"\n\ttop: \"res4b26_branch2c\"\n\tname: \"bn4b26_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b26_branch2c\"\n\ttop: \"res4b26_branch2c\"\n\tname: \"scale4b26_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b25\"\n\tbottom: \"res4b26_branch2c\"\n\ttop: \"res4b26\"\n\tname: \"res4b26\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b26\"\n\ttop: \"res4b26\"\n\tname: \"res4b26_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b26\"\n\ttop: \"res4b27_branch2a\"\n\tname: \"res4b27_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b27_branch2a\"\n\ttop: \"res4b27_branch2a\"\n\tname: \"bn4b27_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b27_branch2a\"\n\ttop: \"res4b27_branch2a\"\n\tname: \"scale4b27_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b27_branch2a\"\n\tbottom: \"res4b27_branch2a\"\n\tname: \"res4b27_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b27_branch2a\"\n\ttop: \"res4b27_branch2b\"\n\tname: \"res4b27_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b27_branch2b\"\n\ttop: \"res4b27_branch2b\"\n\tname: \"bn4b27_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b27_branch2b\"\n\ttop: \"res4b27_branch2b\"\n\tname: \"scale4b27_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b27_branch2b\"\n\tbottom: \"res4b27_branch2b\"\n\tname: \"res4b27_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b27_branch2b\"\n\ttop: \"res4b27_branch2c\"\n\tname: \"res4b27_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b27_branch2c\"\n\ttop: \"res4b27_branch2c\"\n\tname: \"bn4b27_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b27_branch2c\"\n\ttop: \"res4b27_branch2c\"\n\tname: \"scale4b27_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b26\"\n\tbottom: \"res4b27_branch2c\"\n\ttop: \"res4b27\"\n\tname: \"res4b27\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b27\"\n\ttop: \"res4b27\"\n\tname: \"res4b27_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b27\"\n\ttop: \"res4b28_branch2a\"\n\tname: \"res4b28_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b28_branch2a\"\n\ttop: \"res4b28_branch2a\"\n\tname: \"bn4b28_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b28_branch2a\"\n\ttop: \"res4b28_branch2a\"\n\tname: \"scale4b28_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b28_branch2a\"\n\tbottom: \"res4b28_branch2a\"\n\tname: \"res4b28_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b28_branch2a\"\n\ttop: \"res4b28_branch2b\"\n\tname: \"res4b28_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b28_branch2b\"\n\ttop: \"res4b28_branch2b\"\n\tname: \"bn4b28_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b28_branch2b\"\n\ttop: \"res4b28_branch2b\"\n\tname: \"scale4b28_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b28_branch2b\"\n\tbottom: \"res4b28_branch2b\"\n\tname: \"res4b28_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b28_branch2b\"\n\ttop: \"res4b28_branch2c\"\n\tname: \"res4b28_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b28_branch2c\"\n\ttop: \"res4b28_branch2c\"\n\tname: \"bn4b28_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b28_branch2c\"\n\ttop: \"res4b28_branch2c\"\n\tname: \"scale4b28_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b27\"\n\tbottom: \"res4b28_branch2c\"\n\ttop: \"res4b28\"\n\tname: \"res4b28\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b28\"\n\ttop: \"res4b28\"\n\tname: \"res4b28_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b28\"\n\ttop: \"res4b29_branch2a\"\n\tname: \"res4b29_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b29_branch2a\"\n\ttop: \"res4b29_branch2a\"\n\tname: \"bn4b29_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b29_branch2a\"\n\ttop: \"res4b29_branch2a\"\n\tname: \"scale4b29_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b29_branch2a\"\n\tbottom: \"res4b29_branch2a\"\n\tname: \"res4b29_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b29_branch2a\"\n\ttop: \"res4b29_branch2b\"\n\tname: \"res4b29_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b29_branch2b\"\n\ttop: \"res4b29_branch2b\"\n\tname: \"bn4b29_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b29_branch2b\"\n\ttop: \"res4b29_branch2b\"\n\tname: \"scale4b29_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b29_branch2b\"\n\tbottom: \"res4b29_branch2b\"\n\tname: \"res4b29_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b29_branch2b\"\n\ttop: \"res4b29_branch2c\"\n\tname: \"res4b29_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b29_branch2c\"\n\ttop: \"res4b29_branch2c\"\n\tname: \"bn4b29_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b29_branch2c\"\n\ttop: \"res4b29_branch2c\"\n\tname: \"scale4b29_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b28\"\n\tbottom: \"res4b29_branch2c\"\n\ttop: \"res4b29\"\n\tname: \"res4b29\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b29\"\n\ttop: \"res4b29\"\n\tname: \"res4b29_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b29\"\n\ttop: \"res4b30_branch2a\"\n\tname: \"res4b30_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b30_branch2a\"\n\ttop: \"res4b30_branch2a\"\n\tname: \"bn4b30_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b30_branch2a\"\n\ttop: \"res4b30_branch2a\"\n\tname: \"scale4b30_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b30_branch2a\"\n\tbottom: \"res4b30_branch2a\"\n\tname: \"res4b30_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b30_branch2a\"\n\ttop: \"res4b30_branch2b\"\n\tname: \"res4b30_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b30_branch2b\"\n\ttop: \"res4b30_branch2b\"\n\tname: \"bn4b30_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b30_branch2b\"\n\ttop: \"res4b30_branch2b\"\n\tname: \"scale4b30_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b30_branch2b\"\n\tbottom: \"res4b30_branch2b\"\n\tname: \"res4b30_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b30_branch2b\"\n\ttop: \"res4b30_branch2c\"\n\tname: \"res4b30_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b30_branch2c\"\n\ttop: \"res4b30_branch2c\"\n\tname: \"bn4b30_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b30_branch2c\"\n\ttop: \"res4b30_branch2c\"\n\tname: \"scale4b30_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b29\"\n\tbottom: \"res4b30_branch2c\"\n\ttop: \"res4b30\"\n\tname: \"res4b30\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b30\"\n\ttop: \"res4b30\"\n\tname: \"res4b30_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b30\"\n\ttop: \"res4b31_branch2a\"\n\tname: \"res4b31_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b31_branch2a\"\n\ttop: \"res4b31_branch2a\"\n\tname: \"bn4b31_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b31_branch2a\"\n\ttop: \"res4b31_branch2a\"\n\tname: \"scale4b31_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b31_branch2a\"\n\tbottom: \"res4b31_branch2a\"\n\tname: \"res4b31_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b31_branch2a\"\n\ttop: \"res4b31_branch2b\"\n\tname: \"res4b31_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b31_branch2b\"\n\ttop: \"res4b31_branch2b\"\n\tname: \"bn4b31_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b31_branch2b\"\n\ttop: \"res4b31_branch2b\"\n\tname: \"scale4b31_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b31_branch2b\"\n\tbottom: \"res4b31_branch2b\"\n\tname: \"res4b31_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b31_branch2b\"\n\ttop: \"res4b31_branch2c\"\n\tname: \"res4b31_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b31_branch2c\"\n\ttop: \"res4b31_branch2c\"\n\tname: \"bn4b31_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b31_branch2c\"\n\ttop: \"res4b31_branch2c\"\n\tname: \"scale4b31_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b30\"\n\tbottom: \"res4b31_branch2c\"\n\ttop: \"res4b31\"\n\tname: \"res4b31\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b31\"\n\ttop: \"res4b31\"\n\tname: \"res4b31_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b31\"\n\ttop: \"res4b32_branch2a\"\n\tname: \"res4b32_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b32_branch2a\"\n\ttop: \"res4b32_branch2a\"\n\tname: \"bn4b32_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b32_branch2a\"\n\ttop: \"res4b32_branch2a\"\n\tname: \"scale4b32_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b32_branch2a\"\n\tbottom: \"res4b32_branch2a\"\n\tname: \"res4b32_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b32_branch2a\"\n\ttop: \"res4b32_branch2b\"\n\tname: \"res4b32_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b32_branch2b\"\n\ttop: \"res4b32_branch2b\"\n\tname: \"bn4b32_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b32_branch2b\"\n\ttop: \"res4b32_branch2b\"\n\tname: \"scale4b32_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b32_branch2b\"\n\tbottom: \"res4b32_branch2b\"\n\tname: \"res4b32_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b32_branch2b\"\n\ttop: \"res4b32_branch2c\"\n\tname: \"res4b32_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b32_branch2c\"\n\ttop: \"res4b32_branch2c\"\n\tname: \"bn4b32_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b32_branch2c\"\n\ttop: \"res4b32_branch2c\"\n\tname: \"scale4b32_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b31\"\n\tbottom: \"res4b32_branch2c\"\n\ttop: \"res4b32\"\n\tname: \"res4b32\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b32\"\n\ttop: \"res4b32\"\n\tname: \"res4b32_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b32\"\n\ttop: \"res4b33_branch2a\"\n\tname: \"res4b33_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b33_branch2a\"\n\ttop: \"res4b33_branch2a\"\n\tname: \"bn4b33_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b33_branch2a\"\n\ttop: \"res4b33_branch2a\"\n\tname: \"scale4b33_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b33_branch2a\"\n\tbottom: \"res4b33_branch2a\"\n\tname: \"res4b33_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b33_branch2a\"\n\ttop: \"res4b33_branch2b\"\n\tname: \"res4b33_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b33_branch2b\"\n\ttop: \"res4b33_branch2b\"\n\tname: \"bn4b33_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b33_branch2b\"\n\ttop: \"res4b33_branch2b\"\n\tname: \"scale4b33_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b33_branch2b\"\n\tbottom: \"res4b33_branch2b\"\n\tname: \"res4b33_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b33_branch2b\"\n\ttop: \"res4b33_branch2c\"\n\tname: \"res4b33_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b33_branch2c\"\n\ttop: \"res4b33_branch2c\"\n\tname: \"bn4b33_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b33_branch2c\"\n\ttop: \"res4b33_branch2c\"\n\tname: \"scale4b33_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b32\"\n\tbottom: \"res4b33_branch2c\"\n\ttop: \"res4b33\"\n\tname: \"res4b33\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b33\"\n\ttop: \"res4b33\"\n\tname: \"res4b33_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b33\"\n\ttop: \"res4b34_branch2a\"\n\tname: \"res4b34_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b34_branch2a\"\n\ttop: \"res4b34_branch2a\"\n\tname: \"bn4b34_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b34_branch2a\"\n\ttop: \"res4b34_branch2a\"\n\tname: \"scale4b34_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b34_branch2a\"\n\tbottom: \"res4b34_branch2a\"\n\tname: \"res4b34_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b34_branch2a\"\n\ttop: \"res4b34_branch2b\"\n\tname: \"res4b34_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b34_branch2b\"\n\ttop: \"res4b34_branch2b\"\n\tname: \"bn4b34_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b34_branch2b\"\n\ttop: \"res4b34_branch2b\"\n\tname: \"scale4b34_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b34_branch2b\"\n\tbottom: \"res4b34_branch2b\"\n\tname: \"res4b34_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b34_branch2b\"\n\ttop: \"res4b34_branch2c\"\n\tname: \"res4b34_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b34_branch2c\"\n\ttop: \"res4b34_branch2c\"\n\tname: \"bn4b34_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b34_branch2c\"\n\ttop: \"res4b34_branch2c\"\n\tname: \"scale4b34_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b33\"\n\tbottom: \"res4b34_branch2c\"\n\ttop: \"res4b34\"\n\tname: \"res4b34\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b34\"\n\ttop: \"res4b34\"\n\tname: \"res4b34_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b34\"\n\ttop: \"res4b35_branch2a\"\n\tname: \"res4b35_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b35_branch2a\"\n\ttop: \"res4b35_branch2a\"\n\tname: \"bn4b35_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b35_branch2a\"\n\ttop: \"res4b35_branch2a\"\n\tname: \"scale4b35_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b35_branch2a\"\n\tbottom: \"res4b35_branch2a\"\n\tname: \"res4b35_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b35_branch2a\"\n\ttop: \"res4b35_branch2b\"\n\tname: \"res4b35_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b35_branch2b\"\n\ttop: \"res4b35_branch2b\"\n\tname: \"bn4b35_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b35_branch2b\"\n\ttop: \"res4b35_branch2b\"\n\tname: \"scale4b35_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res4b35_branch2b\"\n\tbottom: \"res4b35_branch2b\"\n\tname: \"res4b35_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b35_branch2b\"\n\ttop: \"res4b35_branch2c\"\n\tname: \"res4b35_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b35_branch2c\"\n\ttop: \"res4b35_branch2c\"\n\tname: \"bn4b35_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b35_branch2c\"\n\ttop: \"res4b35_branch2c\"\n\tname: \"scale4b35_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b34\"\n\tbottom: \"res4b35_branch2c\"\n\ttop: \"res4b35\"\n\tname: \"res4b35\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b35\"\n\ttop: \"res4b35\"\n\tname: \"res4b35_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b35\"\n\ttop: \"res5a_branch1\"\n\tname: \"res5a_branch1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 2048\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch1\"\n\ttop: \"res5a_branch1\"\n\tname: \"bn5a_branch1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch1\"\n\ttop: \"res5a_branch1\"\n\tname: \"scale5a_branch1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b35\"\n\ttop: \"res5a_branch2a\"\n\tname: \"res5a_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2a\"\n\ttop: \"res5a_branch2a\"\n\tname: \"bn5a_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2a\"\n\ttop: \"res5a_branch2a\"\n\tname: \"scale5a_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res5a_branch2a\"\n\tbottom: \"res5a_branch2a\"\n\tname: \"res5a_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5a_branch2a\"\n\ttop: \"res5a_branch2b\"\n\tname: \"res5a_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2b\"\n\ttop: \"res5a_branch2b\"\n\tname: \"bn5a_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2b\"\n\ttop: \"res5a_branch2b\"\n\tname: \"scale5a_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res5a_branch2b\"\n\tbottom: \"res5a_branch2b\"\n\tname: \"res5a_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5a_branch2b\"\n\ttop: \"res5a_branch2c\"\n\tname: \"res5a_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 2048\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2c\"\n\ttop: \"res5a_branch2c\"\n\tname: \"bn5a_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2c\"\n\ttop: \"res5a_branch2c\"\n\tname: \"scale5a_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch1\"\n\tbottom: \"res5a_branch2c\"\n\ttop: \"res5a\"\n\tname: \"res5a\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res5a\"\n\ttop: \"res5a\"\n\tname: \"res5a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5a\"\n\ttop: \"res5b_branch2a\"\n\tname: \"res5b_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2a\"\n\ttop: \"res5b_branch2a\"\n\tname: \"bn5b_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2a\"\n\ttop: \"res5b_branch2a\"\n\tname: \"scale5b_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res5b_branch2a\"\n\tbottom: \"res5b_branch2a\"\n\tname: \"res5b_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5b_branch2a\"\n\ttop: \"res5b_branch2b\"\n\tname: \"res5b_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2b\"\n\ttop: \"res5b_branch2b\"\n\tname: \"bn5b_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2b\"\n\ttop: \"res5b_branch2b\"\n\tname: \"scale5b_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res5b_branch2b\"\n\tbottom: \"res5b_branch2b\"\n\tname: \"res5b_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5b_branch2b\"\n\ttop: \"res5b_branch2c\"\n\tname: \"res5b_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 2048\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2c\"\n\ttop: \"res5b_branch2c\"\n\tname: \"bn5b_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2c\"\n\ttop: \"res5b_branch2c\"\n\tname: \"scale5b_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a\"\n\tbottom: \"res5b_branch2c\"\n\ttop: \"res5b\"\n\tname: \"res5b\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res5b\"\n\ttop: \"res5b\"\n\tname: \"res5b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5b\"\n\ttop: \"res5c_branch2a\"\n\tname: \"res5c_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2a\"\n\ttop: \"res5c_branch2a\"\n\tname: \"bn5c_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2a\"\n\ttop: \"res5c_branch2a\"\n\tname: \"scale5c_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res5c_branch2a\"\n\tbottom: \"res5c_branch2a\"\n\tname: \"res5c_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5c_branch2a\"\n\ttop: \"res5c_branch2b\"\n\tname: \"res5c_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2b\"\n\ttop: \"res5c_branch2b\"\n\tname: \"bn5c_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2b\"\n\ttop: \"res5c_branch2b\"\n\tname: \"scale5c_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\ttop: \"res5c_branch2b\"\n\tbottom: \"res5c_branch2b\"\n\tname: \"res5c_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5c_branch2b\"\n\ttop: \"res5c_branch2c\"\n\tname: \"res5c_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 2048\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2c\"\n\ttop: \"res5c_branch2c\"\n\tname: \"bn5c_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2c\"\n\ttop: \"res5c_branch2c\"\n\tname: \"scale5c_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b\"\n\tbottom: \"res5c_branch2c\"\n\ttop: \"res5c\"\n\tname: \"res5c\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res5c\"\n\ttop: \"res5c\"\n\tname: \"res5c_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5c\"\n\ttop: \"pool5\"\n\tname: \"pool5\"\n\ttype: \"Pooling\"\n\tpooling_param {\n\t\tkernel_size: 7\n\t\tstride: 1\n\t\tpool: AVE\n\t}\n}\n\nlayer {\n\tbottom: \"pool5\"\n\ttop: \"fc1000\"\n\tname: \"fc1000\"\n\ttype: \"InnerProduct\"\n\tinner_product_param {\n\t\tnum_output: 1000\n\t}\n}\n\nlayer {\n\tbottom: \"fc1000\"\n\ttop: \"prob\"\n\tname: \"prob\"\n\ttype: \"Softmax\"\n}\n\n" }, { "path": "models/ResNet-50-deploy.prototxt", "content": "name: \"ResNet-50\"\ninput: \"data\"\ninput_dim: 1\ninput_dim: 3\ninput_dim: 224\ninput_dim: 224\n\nlayer {\n\tbottom: \"data\"\n\ttop: \"conv1\"\n\tname: \"conv1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 7\n\t\tpad: 3\n\t\tstride: 2\n\t}\n}\n\nlayer {\n\tbottom: \"conv1\"\n\ttop: \"conv1\"\n\tname: \"bn_conv1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"conv1\"\n\ttop: \"conv1\"\n\tname: \"scale_conv1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"conv1\"\n\ttop: \"conv1\"\n\tname: \"conv1_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"conv1\"\n\ttop: \"pool1\"\n\tname: \"pool1\"\n\ttype: \"Pooling\"\n\tpooling_param {\n\t\tkernel_size: 3\n\t\tstride: 2\n\t\tpool: MAX\n\t}\n}\n\nlayer {\n\tbottom: \"pool1\"\n\ttop: \"res2a_branch1\"\n\tname: \"res2a_branch1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch1\"\n\ttop: \"res2a_branch1\"\n\tname: \"bn2a_branch1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch1\"\n\ttop: \"res2a_branch1\"\n\tname: \"scale2a_branch1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"pool1\"\n\ttop: \"res2a_branch2a\"\n\tname: \"res2a_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2a\"\n\ttop: \"res2a_branch2a\"\n\tname: \"bn2a_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2a\"\n\ttop: \"res2a_branch2a\"\n\tname: \"scale2a_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2a\"\n\ttop: \"res2a_branch2a\"\n\tname: \"res2a_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2a_branch2a\"\n\ttop: \"res2a_branch2b\"\n\tname: \"res2a_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2b\"\n\ttop: \"res2a_branch2b\"\n\tname: \"bn2a_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2b\"\n\ttop: \"res2a_branch2b\"\n\tname: \"scale2a_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2b\"\n\ttop: \"res2a_branch2b\"\n\tname: \"res2a_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2a_branch2b\"\n\ttop: \"res2a_branch2c\"\n\tname: \"res2a_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2c\"\n\ttop: \"res2a_branch2c\"\n\tname: \"bn2a_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch2c\"\n\ttop: \"res2a_branch2c\"\n\tname: \"scale2a_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a_branch1\"\n\tbottom: \"res2a_branch2c\"\n\ttop: \"res2a\"\n\tname: \"res2a\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res2a\"\n\ttop: \"res2a\"\n\tname: \"res2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2a\"\n\ttop: \"res2b_branch2a\"\n\tname: \"res2b_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2a\"\n\ttop: \"res2b_branch2a\"\n\tname: \"bn2b_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2a\"\n\ttop: \"res2b_branch2a\"\n\tname: \"scale2b_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2a\"\n\ttop: \"res2b_branch2a\"\n\tname: \"res2b_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2b_branch2a\"\n\ttop: \"res2b_branch2b\"\n\tname: \"res2b_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2b\"\n\ttop: \"res2b_branch2b\"\n\tname: \"bn2b_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2b\"\n\ttop: \"res2b_branch2b\"\n\tname: \"scale2b_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2b\"\n\ttop: \"res2b_branch2b\"\n\tname: \"res2b_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2b_branch2b\"\n\ttop: \"res2b_branch2c\"\n\tname: \"res2b_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2c\"\n\ttop: \"res2b_branch2c\"\n\tname: \"bn2b_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b_branch2c\"\n\ttop: \"res2b_branch2c\"\n\tname: \"scale2b_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2a\"\n\tbottom: \"res2b_branch2c\"\n\ttop: \"res2b\"\n\tname: \"res2b\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res2b\"\n\ttop: \"res2b\"\n\tname: \"res2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2b\"\n\ttop: \"res2c_branch2a\"\n\tname: \"res2c_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2a\"\n\ttop: \"res2c_branch2a\"\n\tname: \"bn2c_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2a\"\n\ttop: \"res2c_branch2a\"\n\tname: \"scale2c_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2a\"\n\ttop: \"res2c_branch2a\"\n\tname: \"res2c_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2c_branch2a\"\n\ttop: \"res2c_branch2b\"\n\tname: \"res2c_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 64\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2b\"\n\ttop: \"res2c_branch2b\"\n\tname: \"bn2c_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2b\"\n\ttop: \"res2c_branch2b\"\n\tname: \"scale2c_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2b\"\n\ttop: \"res2c_branch2b\"\n\tname: \"res2c_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2c_branch2b\"\n\ttop: \"res2c_branch2c\"\n\tname: \"res2c_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2c\"\n\ttop: \"res2c_branch2c\"\n\tname: \"bn2c_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c_branch2c\"\n\ttop: \"res2c_branch2c\"\n\tname: \"scale2c_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2b\"\n\tbottom: \"res2c_branch2c\"\n\ttop: \"res2c\"\n\tname: \"res2c\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res2c\"\n\ttop: \"res2c\"\n\tname: \"res2c_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res2c\"\n\ttop: \"res3a_branch1\"\n\tname: \"res3a_branch1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch1\"\n\ttop: \"res3a_branch1\"\n\tname: \"bn3a_branch1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch1\"\n\ttop: \"res3a_branch1\"\n\tname: \"scale3a_branch1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res2c\"\n\ttop: \"res3a_branch2a\"\n\tname: \"res3a_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2a\"\n\ttop: \"res3a_branch2a\"\n\tname: \"bn3a_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2a\"\n\ttop: \"res3a_branch2a\"\n\tname: \"scale3a_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2a\"\n\ttop: \"res3a_branch2a\"\n\tname: \"res3a_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3a_branch2a\"\n\ttop: \"res3a_branch2b\"\n\tname: \"res3a_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2b\"\n\ttop: \"res3a_branch2b\"\n\tname: \"bn3a_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2b\"\n\ttop: \"res3a_branch2b\"\n\tname: \"scale3a_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2b\"\n\ttop: \"res3a_branch2b\"\n\tname: \"res3a_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3a_branch2b\"\n\ttop: \"res3a_branch2c\"\n\tname: \"res3a_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2c\"\n\ttop: \"res3a_branch2c\"\n\tname: \"bn3a_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch2c\"\n\ttop: \"res3a_branch2c\"\n\tname: \"scale3a_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a_branch1\"\n\tbottom: \"res3a_branch2c\"\n\ttop: \"res3a\"\n\tname: \"res3a\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3a\"\n\ttop: \"res3a\"\n\tname: \"res3a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3a\"\n\ttop: \"res3b_branch2a\"\n\tname: \"res3b_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b_branch2a\"\n\ttop: \"res3b_branch2a\"\n\tname: \"bn3b_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b_branch2a\"\n\ttop: \"res3b_branch2a\"\n\tname: \"scale3b_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b_branch2a\"\n\ttop: \"res3b_branch2a\"\n\tname: \"res3b_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b_branch2a\"\n\ttop: \"res3b_branch2b\"\n\tname: \"res3b_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b_branch2b\"\n\ttop: \"res3b_branch2b\"\n\tname: \"bn3b_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b_branch2b\"\n\ttop: \"res3b_branch2b\"\n\tname: \"scale3b_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b_branch2b\"\n\ttop: \"res3b_branch2b\"\n\tname: \"res3b_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b_branch2b\"\n\ttop: \"res3b_branch2c\"\n\tname: \"res3b_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3b_branch2c\"\n\ttop: \"res3b_branch2c\"\n\tname: \"bn3b_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b_branch2c\"\n\ttop: \"res3b_branch2c\"\n\tname: \"scale3b_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3a\"\n\tbottom: \"res3b_branch2c\"\n\ttop: \"res3b\"\n\tname: \"res3b\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3b\"\n\ttop: \"res3b\"\n\tname: \"res3b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3b\"\n\ttop: \"res3c_branch2a\"\n\tname: \"res3c_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3c_branch2a\"\n\ttop: \"res3c_branch2a\"\n\tname: \"bn3c_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3c_branch2a\"\n\ttop: \"res3c_branch2a\"\n\tname: \"scale3c_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3c_branch2a\"\n\ttop: \"res3c_branch2a\"\n\tname: \"res3c_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3c_branch2a\"\n\ttop: \"res3c_branch2b\"\n\tname: \"res3c_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3c_branch2b\"\n\ttop: \"res3c_branch2b\"\n\tname: \"bn3c_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3c_branch2b\"\n\ttop: \"res3c_branch2b\"\n\tname: \"scale3c_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3c_branch2b\"\n\ttop: \"res3c_branch2b\"\n\tname: \"res3c_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3c_branch2b\"\n\ttop: \"res3c_branch2c\"\n\tname: \"res3c_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3c_branch2c\"\n\ttop: \"res3c_branch2c\"\n\tname: \"bn3c_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3c_branch2c\"\n\ttop: \"res3c_branch2c\"\n\tname: \"scale3c_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3b\"\n\tbottom: \"res3c_branch2c\"\n\ttop: \"res3c\"\n\tname: \"res3c\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3c\"\n\ttop: \"res3c\"\n\tname: \"res3c_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3c\"\n\ttop: \"res3d_branch2a\"\n\tname: \"res3d_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3d_branch2a\"\n\ttop: \"res3d_branch2a\"\n\tname: \"bn3d_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3d_branch2a\"\n\ttop: \"res3d_branch2a\"\n\tname: \"scale3d_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3d_branch2a\"\n\ttop: \"res3d_branch2a\"\n\tname: \"res3d_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3d_branch2a\"\n\ttop: \"res3d_branch2b\"\n\tname: \"res3d_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 128\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3d_branch2b\"\n\ttop: \"res3d_branch2b\"\n\tname: \"bn3d_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3d_branch2b\"\n\ttop: \"res3d_branch2b\"\n\tname: \"scale3d_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3d_branch2b\"\n\ttop: \"res3d_branch2b\"\n\tname: \"res3d_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3d_branch2b\"\n\ttop: \"res3d_branch2c\"\n\tname: \"res3d_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res3d_branch2c\"\n\ttop: \"res3d_branch2c\"\n\tname: \"bn3d_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3d_branch2c\"\n\ttop: \"res3d_branch2c\"\n\tname: \"scale3d_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3c\"\n\tbottom: \"res3d_branch2c\"\n\ttop: \"res3d\"\n\tname: \"res3d\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res3d\"\n\ttop: \"res3d\"\n\tname: \"res3d_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res3d\"\n\ttop: \"res4a_branch1\"\n\tname: \"res4a_branch1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch1\"\n\ttop: \"res4a_branch1\"\n\tname: \"bn4a_branch1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch1\"\n\ttop: \"res4a_branch1\"\n\tname: \"scale4a_branch1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res3d\"\n\ttop: \"res4a_branch2a\"\n\tname: \"res4a_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2a\"\n\ttop: \"res4a_branch2a\"\n\tname: \"bn4a_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2a\"\n\ttop: \"res4a_branch2a\"\n\tname: \"scale4a_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2a\"\n\ttop: \"res4a_branch2a\"\n\tname: \"res4a_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4a_branch2a\"\n\ttop: \"res4a_branch2b\"\n\tname: \"res4a_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2b\"\n\ttop: \"res4a_branch2b\"\n\tname: \"bn4a_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2b\"\n\ttop: \"res4a_branch2b\"\n\tname: \"scale4a_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2b\"\n\ttop: \"res4a_branch2b\"\n\tname: \"res4a_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4a_branch2b\"\n\ttop: \"res4a_branch2c\"\n\tname: \"res4a_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2c\"\n\ttop: \"res4a_branch2c\"\n\tname: \"bn4a_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch2c\"\n\ttop: \"res4a_branch2c\"\n\tname: \"scale4a_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a_branch1\"\n\tbottom: \"res4a_branch2c\"\n\ttop: \"res4a\"\n\tname: \"res4a\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4a\"\n\ttop: \"res4a\"\n\tname: \"res4a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4a\"\n\ttop: \"res4b_branch2a\"\n\tname: \"res4b_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b_branch2a\"\n\ttop: \"res4b_branch2a\"\n\tname: \"bn4b_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b_branch2a\"\n\ttop: \"res4b_branch2a\"\n\tname: \"scale4b_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b_branch2a\"\n\ttop: \"res4b_branch2a\"\n\tname: \"res4b_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b_branch2a\"\n\ttop: \"res4b_branch2b\"\n\tname: \"res4b_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b_branch2b\"\n\ttop: \"res4b_branch2b\"\n\tname: \"bn4b_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b_branch2b\"\n\ttop: \"res4b_branch2b\"\n\tname: \"scale4b_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b_branch2b\"\n\ttop: \"res4b_branch2b\"\n\tname: \"res4b_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b_branch2b\"\n\ttop: \"res4b_branch2c\"\n\tname: \"res4b_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4b_branch2c\"\n\ttop: \"res4b_branch2c\"\n\tname: \"bn4b_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b_branch2c\"\n\ttop: \"res4b_branch2c\"\n\tname: \"scale4b_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4a\"\n\tbottom: \"res4b_branch2c\"\n\ttop: \"res4b\"\n\tname: \"res4b\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4b\"\n\ttop: \"res4b\"\n\tname: \"res4b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4b\"\n\ttop: \"res4c_branch2a\"\n\tname: \"res4c_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4c_branch2a\"\n\ttop: \"res4c_branch2a\"\n\tname: \"bn4c_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4c_branch2a\"\n\ttop: \"res4c_branch2a\"\n\tname: \"scale4c_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4c_branch2a\"\n\ttop: \"res4c_branch2a\"\n\tname: \"res4c_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4c_branch2a\"\n\ttop: \"res4c_branch2b\"\n\tname: \"res4c_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4c_branch2b\"\n\ttop: \"res4c_branch2b\"\n\tname: \"bn4c_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4c_branch2b\"\n\ttop: \"res4c_branch2b\"\n\tname: \"scale4c_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4c_branch2b\"\n\ttop: \"res4c_branch2b\"\n\tname: \"res4c_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4c_branch2b\"\n\ttop: \"res4c_branch2c\"\n\tname: \"res4c_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4c_branch2c\"\n\ttop: \"res4c_branch2c\"\n\tname: \"bn4c_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4c_branch2c\"\n\ttop: \"res4c_branch2c\"\n\tname: \"scale4c_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4b\"\n\tbottom: \"res4c_branch2c\"\n\ttop: \"res4c\"\n\tname: \"res4c\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4c\"\n\ttop: \"res4c\"\n\tname: \"res4c_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4c\"\n\ttop: \"res4d_branch2a\"\n\tname: \"res4d_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4d_branch2a\"\n\ttop: \"res4d_branch2a\"\n\tname: \"bn4d_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4d_branch2a\"\n\ttop: \"res4d_branch2a\"\n\tname: \"scale4d_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4d_branch2a\"\n\ttop: \"res4d_branch2a\"\n\tname: \"res4d_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4d_branch2a\"\n\ttop: \"res4d_branch2b\"\n\tname: \"res4d_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4d_branch2b\"\n\ttop: \"res4d_branch2b\"\n\tname: \"bn4d_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4d_branch2b\"\n\ttop: \"res4d_branch2b\"\n\tname: \"scale4d_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4d_branch2b\"\n\ttop: \"res4d_branch2b\"\n\tname: \"res4d_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4d_branch2b\"\n\ttop: \"res4d_branch2c\"\n\tname: \"res4d_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4d_branch2c\"\n\ttop: \"res4d_branch2c\"\n\tname: \"bn4d_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4d_branch2c\"\n\ttop: \"res4d_branch2c\"\n\tname: \"scale4d_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4c\"\n\tbottom: \"res4d_branch2c\"\n\ttop: \"res4d\"\n\tname: \"res4d\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4d\"\n\ttop: \"res4d\"\n\tname: \"res4d_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4d\"\n\ttop: \"res4e_branch2a\"\n\tname: \"res4e_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4e_branch2a\"\n\ttop: \"res4e_branch2a\"\n\tname: \"bn4e_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4e_branch2a\"\n\ttop: \"res4e_branch2a\"\n\tname: \"scale4e_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4e_branch2a\"\n\ttop: \"res4e_branch2a\"\n\tname: \"res4e_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4e_branch2a\"\n\ttop: \"res4e_branch2b\"\n\tname: \"res4e_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4e_branch2b\"\n\ttop: \"res4e_branch2b\"\n\tname: \"bn4e_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4e_branch2b\"\n\ttop: \"res4e_branch2b\"\n\tname: \"scale4e_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4e_branch2b\"\n\ttop: \"res4e_branch2b\"\n\tname: \"res4e_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4e_branch2b\"\n\ttop: \"res4e_branch2c\"\n\tname: \"res4e_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4e_branch2c\"\n\ttop: \"res4e_branch2c\"\n\tname: \"bn4e_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4e_branch2c\"\n\ttop: \"res4e_branch2c\"\n\tname: \"scale4e_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4d\"\n\tbottom: \"res4e_branch2c\"\n\ttop: \"res4e\"\n\tname: \"res4e\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4e\"\n\ttop: \"res4e\"\n\tname: \"res4e_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4e\"\n\ttop: \"res4f_branch2a\"\n\tname: \"res4f_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4f_branch2a\"\n\ttop: \"res4f_branch2a\"\n\tname: \"bn4f_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4f_branch2a\"\n\ttop: \"res4f_branch2a\"\n\tname: \"scale4f_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4f_branch2a\"\n\ttop: \"res4f_branch2a\"\n\tname: \"res4f_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4f_branch2a\"\n\ttop: \"res4f_branch2b\"\n\tname: \"res4f_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 256\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4f_branch2b\"\n\ttop: \"res4f_branch2b\"\n\tname: \"bn4f_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4f_branch2b\"\n\ttop: \"res4f_branch2b\"\n\tname: \"scale4f_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4f_branch2b\"\n\ttop: \"res4f_branch2b\"\n\tname: \"res4f_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4f_branch2b\"\n\ttop: \"res4f_branch2c\"\n\tname: \"res4f_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 1024\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res4f_branch2c\"\n\ttop: \"res4f_branch2c\"\n\tname: \"bn4f_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4f_branch2c\"\n\ttop: \"res4f_branch2c\"\n\tname: \"scale4f_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4e\"\n\tbottom: \"res4f_branch2c\"\n\ttop: \"res4f\"\n\tname: \"res4f\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res4f\"\n\ttop: \"res4f\"\n\tname: \"res4f_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res4f\"\n\ttop: \"res5a_branch1\"\n\tname: \"res5a_branch1\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 2048\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch1\"\n\ttop: \"res5a_branch1\"\n\tname: \"bn5a_branch1\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch1\"\n\ttop: \"res5a_branch1\"\n\tname: \"scale5a_branch1\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res4f\"\n\ttop: \"res5a_branch2a\"\n\tname: \"res5a_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 2\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2a\"\n\ttop: \"res5a_branch2a\"\n\tname: \"bn5a_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2a\"\n\ttop: \"res5a_branch2a\"\n\tname: \"scale5a_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2a\"\n\ttop: \"res5a_branch2a\"\n\tname: \"res5a_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5a_branch2a\"\n\ttop: \"res5a_branch2b\"\n\tname: \"res5a_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2b\"\n\ttop: \"res5a_branch2b\"\n\tname: \"bn5a_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2b\"\n\ttop: \"res5a_branch2b\"\n\tname: \"scale5a_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2b\"\n\ttop: \"res5a_branch2b\"\n\tname: \"res5a_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5a_branch2b\"\n\ttop: \"res5a_branch2c\"\n\tname: \"res5a_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 2048\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2c\"\n\ttop: \"res5a_branch2c\"\n\tname: \"bn5a_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch2c\"\n\ttop: \"res5a_branch2c\"\n\tname: \"scale5a_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a_branch1\"\n\tbottom: \"res5a_branch2c\"\n\ttop: \"res5a\"\n\tname: \"res5a\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res5a\"\n\ttop: \"res5a\"\n\tname: \"res5a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5a\"\n\ttop: \"res5b_branch2a\"\n\tname: \"res5b_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2a\"\n\ttop: \"res5b_branch2a\"\n\tname: \"bn5b_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2a\"\n\ttop: \"res5b_branch2a\"\n\tname: \"scale5b_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2a\"\n\ttop: \"res5b_branch2a\"\n\tname: \"res5b_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5b_branch2a\"\n\ttop: \"res5b_branch2b\"\n\tname: \"res5b_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2b\"\n\ttop: \"res5b_branch2b\"\n\tname: \"bn5b_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2b\"\n\ttop: \"res5b_branch2b\"\n\tname: \"scale5b_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2b\"\n\ttop: \"res5b_branch2b\"\n\tname: \"res5b_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5b_branch2b\"\n\ttop: \"res5b_branch2c\"\n\tname: \"res5b_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 2048\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2c\"\n\ttop: \"res5b_branch2c\"\n\tname: \"bn5b_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b_branch2c\"\n\ttop: \"res5b_branch2c\"\n\tname: \"scale5b_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5a\"\n\tbottom: \"res5b_branch2c\"\n\ttop: \"res5b\"\n\tname: \"res5b\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res5b\"\n\ttop: \"res5b\"\n\tname: \"res5b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5b\"\n\ttop: \"res5c_branch2a\"\n\tname: \"res5c_branch2a\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2a\"\n\ttop: \"res5c_branch2a\"\n\tname: \"bn5c_branch2a\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2a\"\n\ttop: \"res5c_branch2a\"\n\tname: \"scale5c_branch2a\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2a\"\n\ttop: \"res5c_branch2a\"\n\tname: \"res5c_branch2a_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5c_branch2a\"\n\ttop: \"res5c_branch2b\"\n\tname: \"res5c_branch2b\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 512\n\t\tkernel_size: 3\n\t\tpad: 1\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2b\"\n\ttop: \"res5c_branch2b\"\n\tname: \"bn5c_branch2b\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2b\"\n\ttop: \"res5c_branch2b\"\n\tname: \"scale5c_branch2b\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2b\"\n\ttop: \"res5c_branch2b\"\n\tname: \"res5c_branch2b_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5c_branch2b\"\n\ttop: \"res5c_branch2c\"\n\tname: \"res5c_branch2c\"\n\ttype: \"Convolution\"\n\tconvolution_param {\n\t\tnum_output: 2048\n\t\tkernel_size: 1\n\t\tpad: 0\n\t\tstride: 1\n\t\tbias_term: false\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2c\"\n\ttop: \"res5c_branch2c\"\n\tname: \"bn5c_branch2c\"\n\ttype: \"BatchNorm\"\n\tbatch_norm_param {\n\t\tuse_global_stats: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5c_branch2c\"\n\ttop: \"res5c_branch2c\"\n\tname: \"scale5c_branch2c\"\n\ttype: \"Scale\"\n\tscale_param {\n\t\tbias_term: true\n\t}\n}\n\nlayer {\n\tbottom: \"res5b\"\n\tbottom: \"res5c_branch2c\"\n\ttop: \"res5c\"\n\tname: \"res5c\"\n\ttype: \"Eltwise\"\n}\n\nlayer {\n\tbottom: \"res5c\"\n\ttop: \"res5c\"\n\tname: \"res5c_relu\"\n\ttype: \"ReLU\"\n}\n\nlayer {\n\tbottom: \"res5c\"\n\ttop: \"pool5\"\n\tname: \"pool5\"\n\ttype: \"Pooling\"\n\tpooling_param {\n\t\tkernel_size: 7\n\t\tstride: 1\n\t\tpool: AVE\n\t}\n}\n\nlayer {\n\tbottom: \"pool5\"\n\ttop: \"fc1000\"\n\tname: \"fc1000\"\n\ttype: \"InnerProduct\"\n\tinner_product_param {\n\t\tnum_output: 1000\n\t}\n}\n\nlayer {\n\tbottom: \"fc1000\"\n\ttop: \"prob\"\n\tname: \"prob\"\n\ttype: \"Softmax\"\n}\n\n" }, { "path": "models/mps_builder", "content": "#!/bin/bash -e\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\ncleanup() {\n kill $(jobs -p) ||:\n echo quit | nvidia-cuda-mps-control > /dev/null 2>&1 ||:\n}\ntrap \"cleanup\" EXIT SIGINT SIGTERM\n\nactive_sms=${1:-100}\necho \"Setting Active SM Percentage: ${active_sms}\"\nexport CUDA_MPS_ACTIVE_THREAD_PERCENTAGE=${active_sms}\n\nnvidia-cuda-mps-control -d ||:\n\nsleep 1\n\necho\necho \"Starting a new shell with MPS running...\"\nbash --rcfile <(echo \"PS1='MPS Subshell: '\")\n" }, { "path": "models/onnx/common.py", "content": "import os\nimport argparse\nimport numpy as np\nimport pycuda.driver as cuda\nimport tensorrt as trt\n\ntry:\n # Sometimes python2 does not understand FileNotFoundError\n FileNotFoundError\nexcept NameError:\n FileNotFoundError = IOError\n\ndef GiB(val):\n return val * 1 << 30\n\ndef find_sample_data(description=\"Runs a TensorRT Python sample\", subfolder=\"\", find_files=[]):\n '''\n Parses sample arguments.\n Args:\n description (str): Description of the sample.\n subfolder (str): The subfolder containing data relevant to this sample\n find_files (str): A list of filenames to find. Each filename will be replaced with an absolute path.\n Returns:\n str: Path of data directory.\n Raises:\n FileNotFoundError\n '''\n kDEFAULT_DATA_ROOT = os.path.abspath(\"/usr/src/tensorrt/data\")\n parser = argparse.ArgumentParser(description=description)\n # Standard command-line arguments for all samples.\n parser.add_argument(\"-d\", \"--datadir\", help=\"Location of the TensorRT sample data directory.\")\n args, unknown_args = parser.parse_known_args()\n # If data directory is not specified, use the default.\n data_root = args.datadir if args.datadir else kDEFAULT_DATA_ROOT\n data_path = os.path.join(data_root, subfolder) if subfolder else data_root\n # Make sure data directory exists.\n if not (os.path.exists(data_path)):\n raise FileNotFoundError(data_path + \" does not exist. Please provide the correct data path with the -d option.\")\n # Find all requested files.\n for index, f in enumerate(find_files):\n find_files[index] = os.path.abspath(os.path.join(data_path, f))\n if not os.path.exists(find_files[index]):\n raise FileNotFoundError(find_files[index] + \" does not exist. Please provide the correct data path with the -d option.\")\n if find_files:\n return data_path, find_files\n else:\n return data_path\n\n# Simple helper data class that's a little nicer to use than a 2-tuple.\nclass HostDeviceMem(object):\n def __init__(self, host_mem, device_mem):\n self.host = host_mem\n self.device = device_mem\n\n def __str__(self):\n return \"Host:\\n\" + str(self.host) + \"\\nDevice:\\n\" + str(self.device)\n\n def __repr__(self):\n return self.__str__()\n\n# Allocates all buffers required for an engine, i.e. host/device inputs/outputs.\ndef allocate_buffers(engine):\n inputs = []\n outputs = []\n bindings = []\n stream = cuda.Stream()\n for binding in engine:\n size = trt.volume(engine.get_binding_shape(binding))\n dtype = trt.nptype(engine.get_binding_dtype(binding))\n # Allocate host and device buffers\n host_mem = cuda.pagelocked_empty(size, dtype)\n device_mem = cuda.mem_alloc(host_mem.nbytes)\n # Append the device buffer to device bindings.\n bindings.append(int(device_mem))\n # Append to the appropriate list.\n if engine.binding_is_input(binding):\n inputs.append(HostDeviceMem(host_mem, device_mem))\n else:\n outputs.append(HostDeviceMem(host_mem, device_mem))\n return inputs, outputs, bindings, stream\n\n# This function is generalized for multiple inputs/outputs.\n# inputs and outputs are expected to be lists of HostDeviceMem objects.\ndef do_inference(context, bindings, inputs, outputs, stream):\n # Transfer input data to the GPU.\n [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs]\n # Run inference.\n context.execute_async(bindings=bindings, stream_handle=stream.handle)\n # Transfer predictions back from the GPU.\n [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs]\n # Synchronize the stream\n stream.synchronize()\n # Return only the host outputs.\n return [out.host for out in outputs]\n" }, { "path": "models/onnx/mnist-v1.3/test_data_set_0/output_0.pb", "content": "\b\u0001\b\n\u0010\u0001J(sDU\u001aČtE\u0001'DWQeYQ\u0018\u00123vNKB\u0003" }, { "path": "models/onnx/mnist-v1.3/test_data_set_1/output_0.pb", "content": "\b\u0001\b\n\u0010\u0001J(\u001cE\f\u000e_\u0001;ÁXy\u0019Ą*_DHԺÓ!9Z" }, { "path": "models/onnx/mnist-v1.3/test_data_set_2/output_0.pb", "content": "\b\u0001\b\n\u0010\u0001J(l\u0011\u00034ĐDM\u000fW\u001fD\u001b˿>'\u0002a¤&B\u00156hE" }, { "path": "models/onnx/onnx_builder.py", "content": "# This sample uses an ONNX ResNet50 Model to create a TensorRT Inference Engine\nimport random\nfrom PIL import Image\nimport numpy as np\nimport ctypes\n\nimport pycuda.driver as cuda\n# This import causes pycuda to automatically manage CUDA context creation and cleanup.\nimport pycuda.autoinit\n\nimport tensorrt as trt\n\nimport sys, os\nsys.path.insert(1, os.path.join(sys.path[0], \".\"))\nimport common\n\ndef softmax(X, theta = 1.0, axis = None):\n \"\"\"\n Compute the softmax of each element along an axis of X.\n\n Parameters\n ----------\n X: ND-Array. Probably should be floats. \n theta (optional): float parameter, used as a multiplier\n prior to exponentiation. Default = 1.0\n axis (optional): axis to compute values along. Default is the \n first non-singleton axis.\n\n Returns an array the same size as X. The result will sum to 1\n along the specified axis.\n \"\"\"\n\n # make X at least 2d\n y = np.atleast_2d(X)\n\n # find axis\n if axis is None:\n axis = next(j[0] for j in enumerate(y.shape) if j[1] > 1)\n\n # multiply y against the theta parameter, \n y = y * float(theta)\n\n # subtract the max for numerical stability\n y = y - np.expand_dims(np.max(y, axis = axis), axis)\n \n # exponentiate y\n y = np.exp(y)\n\n # take the sum along the specified axis\n ax_sum = np.expand_dims(np.sum(y, axis = axis), axis)\n\n # finally: divide elementwise\n p = y / ax_sum\n\n # flatten if X was 1D\n if len(X.shape) == 1: p = p.flatten()\n\n return p\n\nclass ModelData(object):\n MODEL_PATH = \"/work/models/flowers-152.onnx\"\n INPUT_SHAPE = (3, 224, 224)\n # We can convert TensorRT data types to numpy types with trt.nptype()\n DTYPE = trt.float32\n\n# You can set the logger severity higher to suppress messages (or lower to display more messages).\nTRT_LOGGER = trt.Logger(trt.Logger.WARNING)\n\n# Allocate host and device buffers, and create a stream.\ndef allocate_buffers(engine):\n # Determine dimensions and create page-locked memory buffers (i.e. won't be swapped to disk) to hold host inputs/outputs.\n h_input = cuda.pagelocked_empty(trt.volume(engine.get_binding_shape(0)), dtype=trt.nptype(ModelData.DTYPE))\n h_output = cuda.pagelocked_empty(trt.volume(engine.get_binding_shape(1)), dtype=trt.nptype(ModelData.DTYPE))\n # Allocate device memory for inputs and outputs.\n d_input = cuda.mem_alloc(h_input.nbytes)\n d_output = cuda.mem_alloc(h_output.nbytes)\n # Create a stream in which to copy inputs/outputs and run inference.\n stream = cuda.Stream()\n return h_input, d_input, h_output, d_output, stream\n\ndef do_inference(context, h_input, d_input, h_output, d_output, stream):\n # Transfer input data to the GPU.\n cuda.memcpy_htod_async(d_input, h_input, stream)\n # Run inference.\n context.execute_async(bindings=[int(d_input), int(d_output)], stream_handle=stream.handle)\n # Transfer predictions back from the GPU.\n cuda.memcpy_dtoh_async(h_output, d_output, stream)\n # Synchronize the stream\n stream.synchronize()\n\n# The Onnx path is used for Onnx models.\ndef build_engine_onnx(model_file, calibrator=None):\n with trt.Builder(TRT_LOGGER) as builder, builder.create_network() as network, trt.OnnxParser(network, TRT_LOGGER) as parser:\n builder.max_workspace_size = common.GiB(1)\n builder.max_batch_size = 8\n precision = \"fp32\"\n if calibrator:\n builder.int8_mode = True\n builder.int8_calibrator = calibrator\n precision = \"int8\"\n else:\n builder.fp16_mode = True\n precision = \"fp16\"\n # Load the Onnx model and parse it in order to populate the TensorRT network.\n with open(model_file, 'rb') as model:\n parser.parse(model.read())\n engine = builder.build_cuda_engine(network)\n serialized = engine.serialize()\n with open(\"/work/models/flowers-152-b{}-{}.engine\".format(builder.max_batch_size, precision), \"wb\") as file:\n file.write(serialized)\n return engine\n\ndef normalize_image(image_name):\n image = Image.open(image_name)\n # Resize, antialias and transpose the image to CHW.\n c, h, w = ModelData.INPUT_SHAPE\n image_arr = np.asarray(image.resize((w, h), Image.ANTIALIAS)).transpose([2, 0, 1]).astype(trt.nptype(ModelData.DTYPE)).ravel()\n # This particular ResNet50 model requires some preprocessing, specifically, mean normalization.\n return ((image_arr / 255.0) - 0.5) * 2.0\n\ndef load_normalized_test_case(test_image, pagelocked_buffer):\n # Normalize the image and copy to pagelocked memory.\n np.copyto(pagelocked_buffer, normalize_image(test_image))\n return test_image\n\ndef create_calibration_dataset():\n jpegs = []\n for dirpath, subdirs, files in os.walk(\"/work/models/flowers-data/flowers\"):\n for f in files:\n if f.endswith(\"jpg\"):\n jpegs.append(os.path.join(dirpath, f))\n random.shuffle(jpegs)\n return jpegs[:200]\n\nclass ImageBatchStream:\n def __init__(self, batch_size, calibration_files):\n c, h, w = ModelData.INPUT_SHAPE\n self.batch_size = batch_size\n self.files = calibration_files\n self.batch = 0\n self.max_batches = (len(calibration_files) // batch_size) + \\\n (1 if (len(calibration_files) % batch_size) else 0)\n self.calibration_data = np.zeros((batch_size, c, h, w), dtype=np.float32)\n\n def reset(self):\n self.batch = 0\n \n def next_batch(self):\n c, h, w = ModelData.INPUT_SHAPE\n if self.batch < self.max_batches:\n imgs = []\n files_for_batch = self.files[self.batch_size * self.batch : \\\n self.batch_size * (self.batch + 1)]\n for f in files_for_batch:\n print(\"[ImageBatchStream] Processing \", f)\n img = normalize_image(f)\n imgs.append(img.reshape((c, h, w)))\n for i in range(len(imgs)):\n self.calibration_data[i] = imgs[i]\n self.batch += 1\n return np.ascontiguousarray(self.calibration_data, dtype=np.float32)\n else:\n return np.array([])\n\n\nclass MyEntropyCalibrator(trt.IInt8EntropyCalibrator):\n def __init__(self, stream):\n trt.IInt8EntropyCalibrator.__init__(self)\n self.batchstream = stream\n self.d_input = cuda.mem_alloc(self.batchstream.calibration_data.nbytes)\n stream.reset()\n\n def get_batch_size(self):\n return self.batchstream.batch_size\n\n def get_batch(self, bindings, names):\n batch = self.batchstream.next_batch()\n if not batch.size:\n return None\n cuda.memcpy_htod(self.d_input, batch)\n bindings[0] = int(self.d_input)\n return bindings\n\n def read_calibration_cache(self, length):\n return None\n\n def write_calibration_cache(self, ptr, size):\n# cache = ctypes.c_char_p(int(ptr))\n# with open('calibration_cache.bin', 'wb') as f:\n# f.write(cache.value)\n return None\n\n\n\ndef main():\n calibration_files = create_calibration_dataset()\n batch_stream = ImageBatchStream(8, calibration_files)\n int8_calibrator = None\n int8_calibrator = MyEntropyCalibrator(batch_stream)\n engine = build_engine_onnx(\"/work/models/flowers-152.onnx\", calibrator=int8_calibrator)\n# serialized = engine.serialize()\n# with open(\"/work/models/flowers-152-b8-int8.engine\", \"wb\") as file:\n# file.write(serialized)\n# h_input, d_input, h_output, d_output, stream = allocate_buffers(engine)\n# with engine.create_execution_context() as context:\n# for test_image in [\"/work/models/flowers-data/test/image_07927.jpg\",\n# \"/work/models/flowers-data/test/image_06969.jpg\",]:\n# #test_image = \"/work/models/flowers-data/test/image_07927.jpg\" # 13 - blanket flower\n# #test_image = \"/work/models/flowers-data/test/image_06969.jpg\" # 0 - alpine sea holly\n# test_case = load_normalized_test_case(test_image, h_input)\n# do_inference(context, h_input, d_input, h_output, d_output, stream)\n# # We use the highest probability as our prediction. Its index corresponds to the predicted label.\n# pred = np.argmax(h_output)\n# score = softmax(h_output)[pred]\n# print(\"Recognized \" + test_case + \" as \" + str(pred) + \" score: \" + str(score))\n\ndef old_main():\n # Set the data path to the directory that contains the trained models and test images for inference.\n data_path, data_files = common.find_sample_data(description=\"Runs a ResNet50 network with a TensorRT inference engine.\", subfolder=\"resnet50\", find_files=[\"binoculars.jpeg\", \"reflex_camera.jpeg\", \"tabby_tiger_cat.jpg\", ModelData.MODEL_PATH, \"class_labels.txt\"])\n # Get test images, models and labels.\n test_images = data_files[0:3]\n onnx_model_file, labels_file = data_files[3:]\n labels = open(labels_file, 'r').read().split('\\n')\n\n # Build a TensorRT engine.\n with build_engine_onnx(onnx_model_file) as engine:\n # Inference is the same regardless of which parser is used to build the engine, since the model architecture is the same.\n # Allocate buffers and create a CUDA stream.\n h_input, d_input, h_output, d_output, stream = allocate_buffers(engine)\n # Contexts are used to perform inference.\n with engine.create_execution_context() as context:\n # Load a normalized test case into the host input page-locked buffer.\n test_image = random.choice(test_images)\n test_case = load_normalized_test_case(test_image, h_input)\n # Run the engine. The output will be a 1D tensor of length 1000, where each value represents the\n # probability that the image corresponds to that label\n do_inference(context, h_input, d_input, h_output, d_output, stream)\n # We use the highest probability as our prediction. Its index corresponds to the predicted label.\n pred = np.argmax(h_output)\n print(\"Recognized \" + test_case + \" as \" + pred)\n\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "models/setup.py", "content": "#!/usr/bin/env python3\n#\n# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\nimport os\nimport subprocess\n\nmodels = [\n (\"ResNet-50-deploy.prototxt\", \"prob\"),\n# (\"ResNet-152-deploy.prototxt\", \"prob\"),\n]\n\nprecisions = [\n# (\"fp32\", \"\"),\n (\"fp16\", \"--fp16\"),\n# (\"int8\", \"--int8\")\n]\n\ndef main():\n for model, o in models:\n for name, p in precisions:\n for b in [1, 8]: #, 2, 4, 8]:\n n = \"b{}-{}\".format(b, name)\n e = model.replace(\"prototxt\", \"engine\")\n e = e.replace(\"deploy\", n)\n m = os.path.join(\"/work/models\", model)\n if os.path.isfile(e):\n continue\n subprocess.call(\"trtexec --deploy={} --batch={} --output={} {} --engine={}\".format(\n m, b, o, p, e\n ), shell=True)\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "notebooks/Demo Day 1.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import time\\n\",\n \"import numpy as np\\n\",\n \"import wurlitzer\\n\",\n \"\\n\",\n \"import trtlab\\n\",\n \"import infer_test_utils as utils\\n\",\n \"\\n\",\n \"# this allows us to capture stdout and stderr from the backend c++ infer-runtime\\n\",\n \"display_output = wurlitzer.sys_pipes\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"inputs = utils.load_inputs(\\\"/work/models/onnx/mnist-v1.3/test_data_set_0\\\")\\n\",\n \"expected = utils.load_outputs(\\\"/work/models/onnx/mnist-v1.3/test_data_set_0\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!trtexec --onnx=/work/models/onnx/mnist-v1.3/model.onnx --saveEngine=/tmp/mnist-v1.3.engine\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Local Inference Setup\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" manager = trtlab.InferenceManager(max_exec_concurrency=2)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" manager.register_tensorrt_engine(\\\"mnist\\\", \\\"/tmp/mnist-v1.3.engine\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" manager.update_resources()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Local Inference Properties\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"mnist = manager.infer_runner(\\\"mnist\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"mnist.input_bindings()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"mnist.output_bindings()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Local Inference Compute\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"futures = [mnist.infer(Input3=input) for input in inputs]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# free to do other work while inference is being computed\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"results = [f.get() for f in futures]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"for r, e in zip(results, expected):\\n\",\n \" for key, val in r.items():\\n\",\n \" r = val.reshape((1,10))\\n\",\n \" np.testing.assert_almost_equal(r, e, decimal=3)\\n\",\n \" print(\\\"Test Passed\\\")\\n\",\n \" print(\\\"Result: {}\\\".format(np.argmax(utils.softmax(r))))\\n\",\n \"\\n\",\n \"utils.mnist_image(inputs[0]).show()\\n\",\n \"expected[0]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.5.2\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "notebooks/Demo Day 2.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import time\\n\",\n \"import numpy as np\\n\",\n \"import wurlitzer\\n\",\n \"\\n\",\n \"import trtlab\\n\",\n \"import infer_test_utils as utils\\n\",\n \"\\n\",\n \"# this allows us to capture stdout and stderr from the backend c++ infer-runtime\\n\",\n \"display_output = wurlitzer.sys_pipes\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"inputs = utils.load_inputs(\\\"/work/models/onnx/mnist-v1.3/test_data_set_0\\\")\\n\",\n \"expected = utils.load_outputs(\\\"/work/models/onnx/mnist-v1.3/test_data_set_0\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Remote Inference Setup\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" manager = trtlab.RemoteInferenceManager(hostname=\\\"localhost:50052\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" models = manager.get_models()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" print(models)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Remote Inference Properties\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"mnist = manager.infer_runner(\\\"mnist\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"mnist.input_bindings()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"mnist.output_bindings()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Remote Inference Compute\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"futures = [mnist.infer(Input3=input) for input in inputs]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# free to do other work while inference is being computed\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"results = [f.get() for f in futures]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"for r, e in zip(results, expected):\\n\",\n \" for key, val in r.items():\\n\",\n \" r = val.reshape((1,10))\\n\",\n \" np.testing.assert_almost_equal(r, e, decimal=3)\\n\",\n \" print(\\\"Test Passed\\\")\\n\",\n \" print(\\\"Output Binding Name: {}; shape: {}\\\".format(key, val.shape))\\n\",\n \" print(\\\"Result: {}\\\".format(np.argmax(utils.softmax(r))))\\n\",\n \" \\n\",\n \"utils.mnist_image(inputs[0]).show()\\n\",\n \"expected[0]\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.5.2\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "notebooks/Demo Day 3.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import time\\n\",\n \"import numpy as np\\n\",\n \"import wurlitzer\\n\",\n \"\\n\",\n \"import trtlab\\n\",\n \"import infer_test_utils as utils\\n\",\n \"\\n\",\n \"# this allows us to capture stdout and stderr from the backend c++ infer-runtime\\n\",\n \"display_output = wurlitzer.sys_pipes\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!trtexec --onnx=/work/models/onnx/mnist-v1.3/model.onnx --saveEngine=/tmp/mnist-v1.3.engine\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Remote Inference Setup\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" manager = trtlab.InferenceManager(max_exec_concurrency=2)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" manager.register_tensorrt_engine(\\\"mnist\\\", \\\"/tmp/mnist-v1.3.engine\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" manager.update_resources()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Remote Inference Server\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" manager.serve()\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.5.2\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "notebooks/Multiple Models.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import time\\n\",\n \"import numpy as np\\n\",\n \"import wurlitzer\\n\",\n \"\\n\",\n \"import trtlab\\n\",\n \"\\n\",\n \"# this allows us to capture stdout and stderr from the backend c++ infer-runtime\\n\",\n \"display_output = wurlitzer.sys_pipes\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!/work/models/setup.py\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Local Inference Setup\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"WARNING: Logging before InitGoogleLogging() is written to STDERR\\n\",\n \"I0204 22:01:27.543411 925 inference_manager.cc:64] -- Initialzing TensorRT Resource Manager --\\n\",\n \"I0204 22:01:27.543426 925 inference_manager.cc:65] Maximum Execution Concurrency: 4\\n\",\n \"I0204 22:01:27.543429 925 inference_manager.cc:66] Maximum Copy Concurrency: 8\\n\"\n ]\n }\n ],\n \"source\": [\n \"with display_output():\\n\",\n \" manager = trtlab.InferenceManager(max_exec_concurrency=4)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"I0204 22:01:30.164453 925 model.cc:91] Binding: data; isInput: true; dtype size: 4; bytes per batch item: 602112\\n\",\n \"I0204 22:01:30.164479 925 model.cc:91] Binding: prob; isInput: false; dtype size: 4; bytes per batch item: 4000\\n\",\n \"I0204 22:01:30.169529 925 inference_manager.cc:149] -- Registering Model: rn50-b1 --\\n\",\n \"I0204 22:01:30.169546 925 inference_manager.cc:150] Input/Output Tensors require 591.9 KiB\\n\",\n \"I0204 22:01:30.169550 925 inference_manager.cc:151] Execution Activations require 5.7 MiB\\n\",\n \"I0204 22:01:30.169554 925 inference_manager.cc:155] Weights require 75.8 MiB\\n\",\n \"I0204 22:01:30.223752 925 model.cc:91] Binding: data; isInput: true; dtype size: 4; bytes per batch item: 602112\\n\",\n \"I0204 22:01:30.223776 925 model.cc:91] Binding: prob; isInput: false; dtype size: 4; bytes per batch item: 4000\\n\",\n \"I0204 22:01:30.227011 925 inference_manager.cc:149] -- Registering Model: rn50-b8 --\\n\",\n \"I0204 22:01:30.227035 925 inference_manager.cc:150] Input/Output Tensors require 4.6 MiB\\n\",\n \"I0204 22:01:30.227041 925 inference_manager.cc:151] Execution Activations require 39.8 MiB\\n\",\n \"I0204 22:01:30.227046 925 inference_manager.cc:155] Weights require 49.0 MiB\\n\"\n ]\n }\n ],\n \"source\": [\n \"with display_output():\\n\",\n \" manager.register_tensorrt_engine(\\\"rn50-b1\\\", \\\"/work/models/ResNet-50-b1-fp16.engine\\\")\\n\",\n \" manager.register_tensorrt_engine(\\\"rn50-b8\\\", \\\"/work/models/ResNet-50-b8-fp16.engine\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"I0204 22:01:31.025523 925 inference_manager.cc:194] -- Allocating TensorRT Resources --\\n\",\n \"I0204 22:01:31.025539 925 inference_manager.cc:195] Creating 4 TensorRT execution tokens.\\n\",\n \"I0204 22:01:31.025542 925 inference_manager.cc:196] Creating a Pool of 8 Host/Device Memory Stacks\\n\",\n \"I0204 22:01:31.025550 925 inference_manager.cc:197] Each Host Stack contains 4.7 MiB\\n\",\n \"I0204 22:01:31.025554 925 inference_manager.cc:198] Each Device Stack contains 4.8 MiB\\n\",\n \"I0204 22:01:31.025559 925 inference_manager.cc:199] Total GPU Memory: 197.5 MiB\\n\"\n ]\n }\n ],\n \"source\": [\n \"with display_output():\\n\",\n \" manager.update_resources()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Local Inference Properties\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"b1 = manager.infer_runner(\\\"rn50-b1\\\")\\n\",\n \"b8 = manager.infer_runner(\\\"rn50-b8\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 6,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'data': {'dtype': dtype('float32'), 'shape': [3, 224, 224]}}\"\n ]\n },\n \"execution_count\": 6,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"b1.input_bindings()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 7,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"1\"\n ]\n },\n \"execution_count\": 7,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"b1.max_batch_size()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'data': {'dtype': dtype('float32'), 'shape': [3, 224, 224]}}\"\n ]\n },\n \"execution_count\": 8,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"b8.input_bindings()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"8\"\n ]\n },\n \"execution_count\": 9,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"b8.max_batch_size()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"def max_batch_size_shape(x, input='data'):\\n\",\n \" batch = [x.max_batch_size()]\\n\",\n \" batch.extend(x.input_bindings()[input]['shape'])\\n\",\n \" return batch\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"[8, 3, 224, 224]\"\n ]\n },\n \"execution_count\": 12,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"max_batch_size_shape(b8)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Compute\\n\",\n \"\\n\",\n \"Here we launch two async inferences with two different TensorRT engines, one built for batch1, the other for batch8. While these are the same ResNet-50 models, they could be any two unique TensorRT engines.\\n\",\n \"\\n\",\n \"Note: for this example the weights of the model and the input tensors are all random values.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 18,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"futures = [model.infer(data=np.random.random_sample(max_batch_size_shape(model))) for model in [b1, b8]]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 19,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# free to do other work while inference is being computed\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 20,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"results = [f.get() for f in futures]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 26,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"prob binding has shape: (1, 1000, 1, 1)\\n\",\n \"prob binding has shape: (8, 1000, 1, 1)\\n\"\n ]\n }\n ],\n \"source\": [\n \"for result in results:\\n\",\n \" for output, tensor in result.items():\\n\",\n \" print(\\\"{} binding has shape: {}\\\".format(output, tensor.shape))\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.5.2\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "notebooks/Quickstart.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# TensorRT Runtime\\n\",\n \"\\n\",\n \"This example walks through the basic usecase of:\\n\",\n \" 1. initialization the infer-runtime\\n\",\n \" 2. loading a model\\n\",\n \" 3. allocating resources\\n\",\n \" 4. inspecting the input/output bindings of the model\\n\",\n \" 5. evaluating the model using async futures\\n\",\n \" 6. testing for correctness\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import time\\n\",\n \"import numpy as np\\n\",\n \"import wurlitzer\\n\",\n \"\\n\",\n \"import trtlab\\n\",\n \"import infer_test_utils as utils\\n\",\n \"\\n\",\n \"# this allows us to capture stdout and stderr from the backend c++ infer-runtime\\n\",\n \"display_output = wurlitzer.sys_pipes\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!trtexec --onnx=/work/models/onnx/mnist-v1.3/model.onnx --saveEngine=/work/models/onnx/mnist-v1.3/mnist-v1.3.engine\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 1. Initialize infer-runtime\\n\",\n \"\\n\",\n \"The most important option when initializing the infer-runtime is to set the maximum number of conncurrent executions that can be executed at any given time. This value is tunable for your application. Lower setting reduce latency; higher-settings increase throughput. Evaluate how your model performs using ...TODO-this-notebook...\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" models = infer.InferenceManager(max_executions=2)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 2. Register a Model\\n\",\n \"\\n\",\n \"To register a model, simply associate a `model_name` with a path to a TensorRT engine file. The returned object is an `InferRunner` object. Use an `InferRunner` to submit work to the backend inference queue.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" mnist = models.register_tensorrt_engine(\\\"mnist\\\", \\\"/work/models/onnx/mnist-v1.3/mnist-v1.3.engine\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 3. Allocate Resources\\n\",\n \"\\n\",\n \"Before you can submit inference requests, you need to allocate some internal resources. This should be done anytime new models are registered. There maybe a runtime performance interruption if you update the resources while the queue is full.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" models.update_resources()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 4. Inspect Model\\n\",\n \"\\n\",\n \"Query the `InferenceRunner` to see what it expects for inputs and what it will return for outputs.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"mnist.input_bindings()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"mnist.output_bindings()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 5. Submit Infer Requests\\n\",\n \"\\n\",\n \"`InferenceRunner.infer` accecpts a dict of numpy arrays that match the input description, submits this inference request to the backend compute engine and returns a future to a dict of numpy arrays. \\n\",\n \"\\n\",\n \"That means, this method should returns almost immediately; however, that does not mean the inference is complete. Use `get()` to wait for the result. This is a blocking call.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"result = mnist.infer(Input3=np.random.random_sample([1,28,28]))\\n\",\n \"result # result is a future\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"result = result.get()\\n\",\n \"result # result is the value of the future - dict of np arrays\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with display_output():\\n\",\n \" start = time.process_time()\\n\",\n \" result = mnist.infer(**{k: np.random.random_sample(v['shape']) for k,v in mnist.input_bindings().items()})\\n\",\n \" print(\\\"Queue Time: {}\\\".format(time.process_time() - start))\\n\",\n \" result = result.get()\\n\",\n \" print(\\\"Compute Time: {}\\\".format(time.process_time() - start))\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 6. Test for Correctness\\n\",\n \"\\n\",\n \"Load test image and results. [Thanks to the ONNX Model Zoo](https://github.com/onnx/models/tree/master/mnist) for this example.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"inputs = utils.load_inputs(\\\"/work/models/onnx/mnist-v1.3/test_data_set_0\\\")\\n\",\n \"expected = utils.load_outputs(\\\"/work/models/onnx/mnist-v1.3/test_data_set_0\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"utils.mnist_image(inputs[0]).show()\\n\",\n \"expected[0]\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"\\n\",\n \"Submit the images to the inference queue, then wait for each result to be returned.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"results = [mnist.infer(Input3=input) for input in inputs]\\n\",\n \"results = [r.get() for r in results]\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Check results.\\n\",\n \"TODO - update the utils to return dictionaries instead of arrays\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"for r, e in zip(results, expected):\\n\",\n \" for key, val in r.items():\\n\",\n \" r = val.reshape((1,10))\\n\",\n \" np.testing.assert_almost_equal(r, e, decimal=3)\\n\",\n \" print(\\\"Test Passed\\\")\\n\",\n \" print(\\\"Output Binding Name: {}; shape: {}\\\".format(key, val.shape))\\n\",\n \" print(\\\"Result: {}\\\".format(np.argmax(utils.softmax(r))))\\n\",\n \" # r # show the raw tensor\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.5.2\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "notebooks/README.md", "content": "# Jupyter Notebooks\n\nNote: Many of the ONNX examples will fail until TensorRT 5.1 is available in the container.\n\nThis includes:\n - Demo Day 1\n - Demo Day 2\n - Demo Day 3" }, { "path": "requirements.txt", "content": "appdirs==1.4.3\natomicwrites==1.2.1\nattrs==18.2.0\nbackcall==0.1.0\nbleach>=3.1.1\nboto3==1.9.109\nbotocore==1.12.109\nclick==6.7\ncmake==3.11.0\ncycler==0.10.0\ndecorator==4.3.0\ndefusedxml==0.5.0\ndocutils==0.14\nentrypoints==0.3\ngrpcio==1.16.1\nipykernel==5.1.0\nipython==7.3.0\nipython-genutils==0.2.0\njedi==0.13.3\nJinja2==2.10.1\njmespath==0.9.4\njsonschema==3.0.1\njupyter-client==5.2.4\njupyter-core==4.4.0\njupyterlab==0.35.4\njupyterlab-server==0.2.0\nkiwisolver==1.0.1\nMako==1.0.7\nMarkupSafe==1.0\nmatplotlib==3.0.2\nmistune==0.8.4\nmore-itertools==4.3.0\nmxnet==1.4.0.post0\nnbconvert==5.4.1\nnbformat==4.4.0\nnotebook==5.7.8\nnumpy==1.15.4\nonnx==1.3.0\npandocfilters==1.4.2\nparso==0.5.0\npathlib2==2.3.3\npexpect==4.6.0\npickleshare==0.7.5\nPillow>=6.2.2\npluggy==0.8.0\nprometheus-client==0.6.0\nprompt-toolkit==2.0.9\nprotobuf==3.7.0\nptyprocess==0.6.0\npy==1.7.0\npycuda==2018.1.1\nPygments==2.3.1\npyparsing==2.3.1\npyrsistent==0.14.11\npytest==4.0.2\npython-dateutil==2.8.0\npytools==2018.5.2\npyzmq==18.0.0\ns3transfer==0.2.0\nSend2Trash==1.5.0\nsix==1.12.0\nterminado==0.8.1\ntestpath==0.4.2\ntornado==5.1\ntraitlets==4.3.2\ntyping==3.6.6\ntyping-extensions==3.7.2\nurllib3==1.24.2\nwcwidth==0.1.7\nwebencodings==0.5.1\nwurlitzer==1.0.2\n" }, { "path": "trtlab/BUILD.bazel", "content": "exports_files([\n \"core\",\n \"cuda\",\n \"nvrpc\",\n \"tensorrt\",\n ],\n visibility = [\"//visibility:public\"],\n)\n" }, { "path": "trtlab/CMakeLists.txt", "content": "#cmake_minimum_required(VERSION 3.9 FATAL_ERROR)\n\n#project(trtlab)\n\n#include(GNUInstallDirs)\n\n#option(ENABLE_TESTING \"Build tests\" ON)\n\n#set(CMAKE_CXX_STANDARD 17)\n#set(CMAKE_CXX_EXTENSIONS ON)\n\n#set(CMAKE_DEBUG_POSTFIX \"-d\")\nset(CMAKE_POSITION_INDEPENDENT_CODE TRUE)\n#set(default_build_type \"Release\")\n\n#set(CMAKE_THREAD_PREFER_PTHREAD TRUE)\n#find_package(Threads)\n\n#if(ENABLE_TESTING)\n# message(STATUS \"Building Tests\")\n# find_package(GTest) #1.8.1 CONFIG REQUIRED)\n# find_package(benchmark)\n# enable_testing()\n#endif()\n\n# suppress warnings\n\n#add_compile_options(\n# $<$,CXX>,$>:-Wno-deprecated-declarations>\n#)\n\nif(BUILD_MEMORY)\n add_subdirectory(memory)\nendif()\n\nif(BUILD_CORE)\n add_subdirectory(core)\nendif()\n\nif(BUILD_CUDA)\n add_subdirectory(cuda)\nendif()\n\nif(BUILD_NVRPC)\n add_subdirectory(nvrpc)\nendif()\n\nif(BUILD_TENSORRT)\n add_subdirectory(tensorrt)\nendif()\n\nif(BUILD_PYTHON)\n add_subdirectory(pybind)\nendif()\n\n\n# install\n\n# include(CMakePackageConfigHelpers)\n#\n# install(\n# EXPORT ${PROJECT_NAME}-targets\n# NAMESPACE ${PROJECT_NAME}::\n# FILE ${PROJECT_NAME}-targets.cmake\n# DESTINATION \"${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\"\n# )\n#\n# configure_package_config_file(\n# \"${CMAKE_CURRENT_SOURCE_DIR}/cmake/${PROJECT_NAME}-config.cmake.in\"\n# ${PROJECT_NAME}-config.cmake\n# INSTALL_DESTINATION \"${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\"\n# NO_CHECK_REQUIRED_COMPONENTS_MACRO\n# PATH_VARS CMAKE_INSTALL_INCLUDEDIR\n# )\n#\n# install(\n# # FILES \"${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake\"\n# DESTINATION \"${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\"\n# )\n" }, { "path": "trtlab/core/BUILD.bazel", "content": "cc_library(\n name = \"core\",\n srcs = glob([\n \"src/**/*.cc\",\n \"src/**/*.h\",\n ]),\n hdrs = glob(\n [\"include/**/*.h\"],\n ),\n deps = [\n \"@com_google_glog//:glog\",\n ],\n strip_include_prefix = \"include\",\n visibility = [\"//visibility:public\"],\n)\n" }, { "path": "trtlab/core/CMakeLists.txt", "content": "include(GNUInstallDirs)\nset(CMAKE_THREAD_PREFER_PTHREAD TRUE)\nfind_package(Threads)\nfind_package(cpuaff)\nfind_package(glog 0.3.5 REQUIRED)\nfind_package(dlpack)\nfind_package(Boost REQUIRED COMPONENTS fiber_numa fiber context)\n\nset(CMAKE_CXX_STANDARD 17)\nset(CMAKE_CXX_STANDARD_REQUIRED ON)\nset(CMAKE_CXX_EXTENSIONS ON)\n\n# when linking against static glog one also needs to link against gflags\n# which version of gflags is depended on build type\nfind_package(gflags REQUIRED)\nif( CMAKE_BUILD_TYPE STREQUAL \"Debug\" )\n find_package(gflags COMPONENTS nothreads_shared)\n set(trtlab_gflags \"gflags_nothreads_shared\")\n message(STATUS \"gflags: ${trtlab_gflags}\")\n message(STATUS \"build_type: ${CMAKE_BUILD_TYPE}\")\nelse()\n find_package(gflags COMPONENTS nothreads_static)\n set(trtlab_gflags \"gflags_nothreads_static\")\n message(STATUS \"gflags: ${trtlab_gflags}\")\n message(STATUS \"build_type: ${CMAKE_BUILD_TYPE}\")\nendif()\n\nset(header_path ${CMAKE_CURRENT_SOURCE_DIR}/include/trtlab/core)\n\nadd_library(core\n# src/types.cc\n src/affinity.cc\n src/utils.cc\n src/cyclic_buffer.cc\n src/cyclic_windowed_buffer.cc\n)\n\nadd_library(${PROJECT_NAME}::core ALIAS core)\n\nmessage(STATUS \"jemalloc: ${JEMALLOC_STATIC_LIBRARIES}\")\n\ntarget_link_libraries(core\n PUBLIC\n trtlab_memory\n\n dlpack::dlpack\n Threads::Threads\n $<$,$>>:rt>\n glog::glog\n ${trtlab_gflags}\n Boost::fiber\n Boost::fiber_numa\n Boost::context\n ${JEMALLOC_STATIC_LIBRARIES}\n dl\n)\n\ntarget_include_directories(core\n PUBLIC\n $\n PRIVATE\n ${header_path}\n)\n\nset_target_properties(core PROPERTIES OUTPUT_NAME ${PROJECT_NAME}_core)\n\ninstall(\n TARGETS core\n EXPORT ${PROJECT_NAME}-targets\n RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}\n LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}\n ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}\n INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}\n)\n\ninstall(\n DIRECTORY include/\n DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}\n)\n\n#if(ENABLE_TESTING)\n add_subdirectory(tests)\n#endif()\n\n#if(benchmark_FOUND)\n add_subdirectory(benchmarks)\n#endif()\n\n" }, { "path": "trtlab/core/benchmarks/CMakeLists.txt", "content": "# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# * Neither the name of NVIDIA CORPORATION nor the names of its\n# contributors may be used to endorse or promote products derived\n# from this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nfind_package(benchmark)\n\nadd_executable(bench_core\n main.cc\n bench_pool.cc\n bench_thread_pool.cc\n bench_batcher.cc\n# bench_memory.cc\n# bench_memory_stack.cc\n)\n\ntarget_link_libraries(bench_core \n PRIVATE \n ${PROJECT_NAME}::core\n benchmark::benchmark\n)\n\nadd_test(NAME bench_core COMMAND $)\n" }, { "path": "trtlab/core/benchmarks/bench_batcher.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n\n#include \n\n#include \n#include \n\nusing namespace trtlab;\n\nstatic void batcher_standard_batcher_int(benchmark::State& state)\n{\n StandardBatcher batcher(state.range(0));\n std::size_t counter = 0;\n\n for (auto _ : state)\n {\n auto future = batcher.enqueue(++counter);\n auto batch = batcher.update();\n\n if (batch)\n {\n batch->promise.set_value();\n future.wait();\n }\n }\n state.SetItemsProcessed(static_cast(state.iterations()));\n}\n\nstruct audio_state\n{\n const std::uint16_t* data;\n std::size_t size;\n std::shared_ptr state;\n};\n\nstatic void batcher_standard_batcher_audio(benchmark::State& state)\n{\n StandardBatcher batcher(state.range(0));\n std::size_t counter = 0;\n\n for (auto _ : state)\n {\n auto future = batcher.enqueue({nullptr, 0ul, nullptr});\n auto batch = batcher.update();\n\n if (batch)\n {\n batch->promise.set_value();\n future.wait();\n }\n }\n state.SetItemsProcessed(static_cast(state.iterations()));\n}\n\nstatic void batcher_engine(benchmark::State& state)\n{\n const std::size_t batch_size = state.range(0);\n\n auto execute_on_batch = [](const std::vector& batch, std::function free_inputs) { free_inputs(); };\n\n auto thread_pool = std::make_shared(1);\n auto task_pool = std::make_shared();\n StandardBatcher batcher(batch_size);\n Dispatcher dispatcher(std::move(batcher), std::chrono::milliseconds(15), thread_pool, task_pool, execute_on_batch);\n\n std::queue> f;\n\n int pre_load = 3;\n\n for (int i = 0; i < pre_load; i++)\n {\n f.push(dispatcher.enqueue(0));\n }\n for (int i = 0; i < (batch_size - 1) * pre_load; i++)\n {\n dispatcher.enqueue(i);\n }\n\n for (auto _ : state)\n {\n f.push(dispatcher.enqueue(0));\n for (int i = 0; i < batch_size - 1; i++)\n {\n dispatcher.enqueue(i);\n }\n f.front().wait();\n f.pop();\n }\n\n while (!f.empty())\n {\n f.front().wait();\n f.pop();\n }\n\n state.SetItemsProcessed(static_cast(state.iterations()) * state.range(0));\n}\n\nBENCHMARK(batcher_standard_batcher_int)->RangeMultiplier(2)->Range(1, 1 << 7);\nBENCHMARK(batcher_standard_batcher_audio)->RangeMultiplier(2)->Range(1 << 6, 1 << 7);\nBENCHMARK(batcher_engine)->RangeMultiplier(2)->Range(4, 1 << 6)->UseRealTime()->MinTime(3.0);\n;" }, { "path": "trtlab/core/benchmarks/bench_memory.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n\n#include \"trtlab/core/memory/allocator.h\"\n#include \"trtlab/core/memory/malloc.h\"\n#include \"trtlab/core/memory/sysv_allocator.h\"\n\nusing namespace trtlab;\nusing namespace trtlab;\n\n\nstatic void BM_Memory_SystemMalloc(benchmark::State& state)\n{\n for(auto _ : state)\n {\n //auto unique = std::make_unique>(1024 * 1024);\n //auto shared = std::make_shared>(1024 * 1024);\n Allocator memory1(1024 * 1024);\n Allocator memory2(1024 * 1024);\n Allocator memory3(1024 * 1024);\n }\n}\n\nstatic void BM_Memory_SystemV_descriptor(benchmark::State& state)\n{\n auto master = std::make_unique>(1024 * 1024);\n for(auto _ : state)\n {\n auto mdesc = SystemV::Attach(master->ShmID());\n }\n}\n\n/*\nstatic void BM_Memory_HostDescriptor(benchmark::State& state)\n{\n void *ptr = (void*)0xDEADBEEF;\n mem_size_t size = 1337;\n\n for(auto _ : state)\n {\n nextgen::HostDescriptor hdesc(ptr, size, []{});\n }\n}\n*/\n\n/*\nstatic void BM_Memory_SharedHostDescriptor(benchmark::State& state)\n{\n void *ptr = (void*)0xDEADBEEF;\n mem_size_t size = 1337;\n\n for(auto _ : state)\n {\n nextgen::Descriptor hdesc(ptr, size, []{});\n auto shared = std::make_shared>(std::move(hdesc));\n }\n}\n*/\n\n/*\nstatic void BM_Memory_NextGenMalloc(benchmark::State& state)\n{\n static mem_size_t one_mb = 1024*1024;\n\n for(auto _ : state)\n {\n auto hdesc0 = nextgen::Malloc::Allocate(one_mb);\n auto hdesc1 = nextgen::Malloc::Allocate(one_mb);\n }\n}\n*/\n\nBENCHMARK(BM_Memory_SystemMalloc);\nBENCHMARK(BM_Memory_SystemV_descriptor);\n// BENCHMARK(BM_Memory_HostDescriptor);\n// BENCHMARK(BM_Memory_SharedHostDescriptor);\n// BENCHMARK(BM_Memory_NextGenMalloc);" }, { "path": "trtlab/core/benchmarks/bench_memory_stack.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n\n#include \"trtlab/core/hybrid_mutex.h\"\n\n#include \"trtlab/core/memory.h\"\n\n#include \n\nusing namespace trtlab;\nusing namespace trtlab;\n\nstatic void allocators_transactional_raw(benchmark::State& state)\n{\n using namespace memory::literals;\n\n auto alloc = memory::make_allocator_adapter(memory::malloc_allocator());\n auto block_alloc = memory::trtlab::make_growth_capped_block_allocator(1_MiB, 4u, std::move(alloc));\n auto trans_alloc = memory::trtlab::make_transactional_allocator(std::move(block_alloc));\n trans_alloc.reserve_blocks(4u);\n\n for(auto _ : state)\n {\n for(int i=0; i < state.range(0); i++)\n {\n auto ptr = trans_alloc.allocate_node(1024, 64);\n trans_alloc.deallocate_node(ptr, 0u, 0u);\n }\n }\n}\n\nstatic void allocators_transactional_std(benchmark::State& state)\n{\n using namespace memory::literals;\n\n auto alloc = memory::make_allocator_adapter(memory::malloc_allocator());\n auto block_alloc = memory::trtlab::make_growth_capped_block_allocator(1_MiB, 4u, std::move(alloc));\n auto trans_alloc = memory::trtlab::make_transactional_allocator(std::move(block_alloc));\n trans_alloc.reserve_blocks(4u);\n\n auto smart = memory::trtlab::make_allocator(std::move(trans_alloc));\n\n for(auto _ : state)\n {\n for(int i=0; i < state.range(0); i++)\n {\n auto ptr = smart.allocate_node(1024, 64);\n smart.deallocate_node(ptr, 0u, 0u);\n }\n }\n}\n\nstatic void allocators_transactional_md(benchmark::State& state)\n{\n using namespace memory::literals;\n\n auto alloc = memory::make_allocator_adapter(memory::malloc_allocator());\n auto block_alloc = memory::trtlab::make_growth_capped_block_allocator(1_MiB, 4u, std::move(alloc));\n auto trans_alloc = memory::trtlab::make_transactional_allocator(std::move(block_alloc));\n trans_alloc.reserve_blocks(4u);\n\n auto smart = memory::trtlab::make_allocator(std::move(trans_alloc));\n\n for(auto _ : state)\n {\n for(int i=0; i < state.range(0); i++)\n {\n auto md = smart.allocate_descriptor(1024, 64);\n }\n }\n}\n\n#if 0\ntemplate\nusing custom_vector = std::vector>;\n\ntemplate\nauto make_vector(RawAllocator& alloc)\n{\n using std_allocator = memory::std_allocator;\n return std::vector(std_allocator(alloc));\n}\n\n\nstatic void BM_vector_transactional(benchmark::State& state)\n{\n using namespace memory::literals;\n\n auto malloc = memory::make_allocator_reference(memory::MallocAllocator());\n auto block_alloc = memory::trtlab::make_growth_capped_block_allocator(128_MiB, 8, std::move(malloc));\n auto trans_alloc = memory::trtlab::make_transactional_allocator(std::move(block_alloc));\n trans_alloc.reserve_blocks(8);\n\n for(auto _ : state)\n {\n auto vec = make_vector(trans_alloc);\n vec.reserve(1024*1024*8);\n }\n}\n\nstatic void BM_vector_smart_transactional(benchmark::State& state)\n{\n using namespace memory::literals;\n\n auto malloc = memory::make_allocator_reference(memory::MallocAllocator());\n auto block_alloc = memory::trtlab::make_growth_capped_block_allocator(128_MiB, 8, std::move(malloc));\n auto trans_alloc = memory::trtlab::make_transactional_allocator(std::move(block_alloc));\n trans_alloc.reserve_blocks(8);\n\n auto smart = memory::trtlab::make_allocator(std::move(trans_alloc));\n \n for(auto _ : state)\n {\n auto vec = memory::trtlab::make_vector(smart);\n vec.reserve(1024*1024*8);\n }\n}\n\n\nstatic void BM_CyclicAllocator_stl_allocator(benchmark::State& state)\n{\n {\n auto v0 = custom_vector(1024);\n }\n for(auto _ : state)\n {\n custom_vector vector;\n vector.reserve(1024*1024*8);\n }\n}\n\nstatic void BM_CyclicAllocator_stl_allocator2(benchmark::State& state)\n{\n size_t ctr = 1024;\n for(auto _ : state)\n {\n custom_vector v3;\n v3.reserve(ctr*ctr*8);\n }\n}\n\nstatic void BM_vector_default(benchmark::State& state)\n{\n size_t ctr = 1024;\n for(auto _ : state)\n {\n std::vector vec;\n vec.reserve(1024*1024*8);\n }\n}\n\nstatic void BM_stl_allocator_ctor(benchmark::State& state)\n{\n for(auto _ : state)\n {\n auto a = stl::temporary_allocator();\n }\n}\n\nstatic void BM_stl_allocator_allocate_lifecycle(benchmark::State& state)\n{\n for(auto _ : state)\n {\n auto a = stl::temporary_allocator();\n auto i = a.allocate(1024);\n a.deallocate(i, 1024);\n }\n}\n#endif\n\nBENCHMARK(allocators_transactional_raw)->RangeMultiplier(2)->Range(1, 1 << 2);\nBENCHMARK(allocators_transactional_std)->RangeMultiplier(2)->Range(1, 1 << 0);\nBENCHMARK(allocators_transactional_md)->RangeMultiplier(2)->Range(1, 1 << 0);\n" }, { "path": "trtlab/core/benchmarks/bench_pool.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \"trtlab/core/pool.h\"\n#include \"trtlab/core/userspace_threads.h\"\n#include \n\nstatic void BM_Pool_v1_Pop(benchmark::State& state)\n{\n using trtlab::v1::Pool;\n struct Object\n {\n };\n auto pool = Pool::Create();\n pool->EmplacePush(new Object);\n\n for(auto _ : state)\n {\n auto obj = pool->Pop();\n }\n}\n\nstatic void BM_Pool_v2_Pop(benchmark::State& state)\n{\n using trtlab::v2::Pool;\n struct Object\n {\n };\n auto pool = Pool::Create();\n pool->EmplacePush();\n\n for(auto _ : state)\n {\n auto obj = pool->Pop();\n }\n}\n\nstatic void BM_Pool_v3_Pop(benchmark::State& state)\n{\n using trtlab::v3::Pool;\n struct Object\n {\n };\n auto pool = Pool::Create();\n pool->emplace_push();\n\n for(auto _ : state)\n {\n auto obj = std::move(pool->pop());\n }\n}\n\nstatic void BM_Pool_v4_Pop(benchmark::State& state)\n{\n using trtlab::v4::Pool;\n struct Object\n {\n };\n auto pool = Pool::Create();\n pool->EmplacePush();\n\n for(auto _ : state)\n {\n auto obj = std::move(pool->pop_unique());\n }\n}\n\nstatic void BM_Pool_v4_Pop_Shared(benchmark::State& state)\n{\n using trtlab::v4::Pool;\n struct Object\n {\n };\n auto pool = Pool::Create();\n pool->emplace_push();\n\n for(auto _ : state)\n {\n auto obj = std::move(pool->pop_shared());\n }\n}\n\nstatic void BM_Pool_v3_Pop_Userspace(benchmark::State& state)\n{\n using trtlab::v3::Pool;\n struct Object\n {\n };\n auto pool = Pool::Create();\n pool->emplace_push();\n\n for(auto _ : state)\n {\n auto obj = std::move(pool->pop());\n }\n}\n\nBENCHMARK(BM_Pool_v1_Pop);\nBENCHMARK(BM_Pool_v2_Pop);\nBENCHMARK(BM_Pool_v3_Pop);\nBENCHMARK(BM_Pool_v4_Pop);\nBENCHMARK(BM_Pool_v4_Pop_Shared);\nBENCHMARK(BM_Pool_v3_Pop_Userspace);\n" }, { "path": "trtlab/core/benchmarks/bench_thread_pool.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \"trtlab/core/hybrid_condition.h\"\n#include \"trtlab/core/hybrid_mutex.h\"\n#include \"trtlab/core/thread_pool.h\"\n#include \n\nstatic void BM_ThreadPool_Enqueue(benchmark::State& state)\n{\n using trtlab::ThreadPool;\n auto pool = std::make_unique(1);\n\n for(auto _ : state)\n {\n CHECK(pool);\n // enqueue only\n auto future = pool->enqueue([] {});\n //future.get();\n }\n}\nBENCHMARK(BM_ThreadPool_Enqueue)->UseRealTime();\n\nstatic void BM_HybridThreadPool_Enqueue(benchmark::State& state)\n{\n using trtlab::BaseThreadPool;\n auto pool = std::make_unique>(1);\n\n for(auto _ : state)\n {\n auto future = pool->enqueue([] {});\n }\n}\nBENCHMARK(BM_HybridThreadPool_Enqueue)->UseRealTime();" }, { "path": "trtlab/core/benchmarks/main.cc", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#include \n\nBENCHMARK_MAIN();" }, { "path": "trtlab/core/include/trtlab/core/affinity.h", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#pragma once\n#include \n\n#include \"cpuaff/cpuaff.hpp\"\n\nnamespace trtlab\n{\n struct affinity;\n\n class cpu_set final : public cpuaff::cpu_set\n {\n public:\n using cpuaff::cpu_set::cpu_set;\n\n cpu_set get_intersection(const cpu_set& other) const;\n cpu_set get_union(const cpu_set& other) const;\n cpu_set get_difference(const cpu_set& other) const;\n\n static cpu_set from_string(std::string);\n\n std::string cpus_string() const;\n std::string cores_string() const;\n std::string sockets_string() const;\n\n auto get_allocator() const -> cpuaff::round_robin_allocator\n {\n return cpuaff::round_robin_allocator(*this);\n };\n\n friend std::ostream& operator<<(std::ostream& s, const cpu_set& cpus);\n };\n\n std::ostream& operator<<(std::ostream& s, const cpu_set& cpus);\n\n class affinity_guard final\n {\n // hold the original affinity of the calling thread\n // the original affinity will be restored on destruction\n cpu_set m_original_cpus;\n\n public:\n affinity_guard();\n explicit affinity_guard(const cpu_set&);\n ~affinity_guard();\n\n affinity_guard(const affinity_guard&) = delete;\n affinity_guard& operator=(const affinity_guard&) = delete;\n\n affinity_guard(affinity_guard&&) noexcept = delete;\n affinity_guard& operator=(affinity_guard&&) noexcept = delete;\n };\n\n struct numa_node\n {\n unsigned id;\n cpu_set cpus;\n std::vector distances;\n\n friend std::ostream& operator<<(std::ostream& s, const numa_node& cpus);\n };\n\n std::ostream& operator<<(std::ostream& s, const numa_node& cpus);\n\n struct affinity final\n {\n struct this_thread final\n {\n static cpu_set get_affinity();\n static void set_affinity(const cpu_set&);\n };\n\n struct system final\n {\n // static cpu_set cpus_by_numa(int numa_id);\n // static cpu_set cpus_by_socket(int socket_id);\n // static cpu_set cpus_by_core(int core_id);\n // static cpu_set cpus_by_hyperthread(int thread_id);\n\n static cpuaff::cpu cpu_from_logical_id(int id);\n\n static std::vector topology();\n };\n };\n\n} // namespace trtlab\n" }, { "path": "trtlab/core/include/trtlab/core/async_compute.h", "content": "/* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * * Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * * Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * * Neither the name of NVIDIA CORPORATION nor the names of its\n * contributors may be used to endorse or promote products derived\n * from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n#pragma once\n\n#include \n#include \n\nnamespace trtlab\n{\n namespace async\n {\n namespace detail\n {\n template